Add loan-advisor-agent (part 3)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/baseten/chat.py +121 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/base_aws_llm.py +1406 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/handler.py +97 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/transformation.py +549 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/__init__.py +30 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/__init__.py +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/transformation.py +1037 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_handler.py +512 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/handler.py +5 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/transformation.py +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_transformation.py +2126 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_agent/transformation.py +547 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_handler.py +1893 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py +99 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py +75 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +135 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py +80 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py +119 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +266 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +120 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py +192 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +99 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +225 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py +116 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +280 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py +98 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +206 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +613 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/common_utils.py +1140 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/cost_calculation.py +27 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +115 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/handler.py +143 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/transformation.py +272 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_nova_transformation.py +361 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py +88 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py +101 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py +131 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/cohere_transformation.py +47 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/embedding.py +699 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py +304 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/files/handler.py +210 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/files/transformation.py +786 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/__init__.py +9 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/handler.py +309 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/stability_transformation.py +399 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py +220 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py +164 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py +128 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py +160 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/cost_calculator.py +24 -0
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/baseten/chat.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class BasetenConfig(OpenAIGPTConfig):
|
| 6 |
+
"""
|
| 7 |
+
Reference: https://inference.baseten.co/v1
|
| 8 |
+
|
| 9 |
+
Below are the parameters:
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
max_tokens: Optional[int] = None
|
| 13 |
+
response_format: Optional[dict] = None
|
| 14 |
+
seed: Optional[int] = None
|
| 15 |
+
stream: Optional[bool] = None
|
| 16 |
+
top_p: Optional[int] = None
|
| 17 |
+
tool_choice: Optional[str] = None
|
| 18 |
+
tools: Optional[list] = None
|
| 19 |
+
user: Optional[str] = None
|
| 20 |
+
presence_penalty: Optional[int] = None
|
| 21 |
+
frequency_penalty: Optional[int] = None
|
| 22 |
+
stream_options: Optional[dict] = None
|
| 23 |
+
|
| 24 |
+
def __init__(
|
| 25 |
+
self,
|
| 26 |
+
max_tokens: Optional[int] = None,
|
| 27 |
+
response_format: Optional[dict] = None,
|
| 28 |
+
seed: Optional[int] = None,
|
| 29 |
+
stop: Optional[list] = None,
|
| 30 |
+
stream: Optional[bool] = None,
|
| 31 |
+
temperature: Optional[float] = None,
|
| 32 |
+
top_p: Optional[int] = None,
|
| 33 |
+
tool_choice: Optional[str] = None,
|
| 34 |
+
tools: Optional[list] = None,
|
| 35 |
+
user: Optional[str] = None,
|
| 36 |
+
presence_penalty: Optional[int] = None,
|
| 37 |
+
frequency_penalty: Optional[int] = None,
|
| 38 |
+
stream_options: Optional[dict] = None,
|
| 39 |
+
) -> None:
|
| 40 |
+
locals_ = locals().copy()
|
| 41 |
+
for key, value in locals_.items():
|
| 42 |
+
if key != "self" and value is not None:
|
| 43 |
+
setattr(self.__class__, key, value)
|
| 44 |
+
|
| 45 |
+
@classmethod
|
| 46 |
+
def get_config(cls):
|
| 47 |
+
return super().get_config()
|
| 48 |
+
|
| 49 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 50 |
+
"""
|
| 51 |
+
Get the supported OpenAI params for the given model
|
| 52 |
+
"""
|
| 53 |
+
return [
|
| 54 |
+
"max_tokens",
|
| 55 |
+
"max_completion_tokens",
|
| 56 |
+
"response_format",
|
| 57 |
+
"seed",
|
| 58 |
+
"stop",
|
| 59 |
+
"stream",
|
| 60 |
+
"temperature",
|
| 61 |
+
"top_p",
|
| 62 |
+
"tool_choice",
|
| 63 |
+
"tools",
|
| 64 |
+
"user",
|
| 65 |
+
"presence_penalty",
|
| 66 |
+
"frequency_penalty",
|
| 67 |
+
"stream_options",
|
| 68 |
+
]
|
| 69 |
+
|
| 70 |
+
def map_openai_params(
|
| 71 |
+
self,
|
| 72 |
+
non_default_params: dict,
|
| 73 |
+
optional_params: dict,
|
| 74 |
+
model: str,
|
| 75 |
+
drop_params: bool,
|
| 76 |
+
) -> dict:
|
| 77 |
+
supported_openai_params = self.get_supported_openai_params(model=model)
|
| 78 |
+
for param, value in non_default_params.items():
|
| 79 |
+
if param == "max_completion_tokens":
|
| 80 |
+
optional_params["max_tokens"] = value
|
| 81 |
+
elif param in supported_openai_params:
|
| 82 |
+
optional_params[param] = value
|
| 83 |
+
return optional_params
|
| 84 |
+
|
| 85 |
+
def _get_openai_compatible_provider_info(
|
| 86 |
+
self, api_base: str, api_key: str
|
| 87 |
+
) -> tuple:
|
| 88 |
+
"""
|
| 89 |
+
Get the OpenAI compatible provider info for Baseten
|
| 90 |
+
"""
|
| 91 |
+
# Default to Model API
|
| 92 |
+
default_api_base = "https://inference.baseten.co/v1"
|
| 93 |
+
default_api_key = api_key or "BASETEN_API_KEY"
|
| 94 |
+
|
| 95 |
+
return default_api_base, default_api_key
|
| 96 |
+
|
| 97 |
+
@staticmethod
|
| 98 |
+
def is_dedicated_deployment(model: str) -> bool:
|
| 99 |
+
"""
|
| 100 |
+
Check if the model is a dedicated deployment (8-digit alphanumeric code)
|
| 101 |
+
"""
|
| 102 |
+
# Remove 'baseten/' prefix if present
|
| 103 |
+
model_id = model.replace("baseten/", "")
|
| 104 |
+
|
| 105 |
+
# Check if it's an 8-digit alphanumeric code
|
| 106 |
+
import re
|
| 107 |
+
|
| 108 |
+
return bool(re.match(r"^[a-zA-Z0-9]{8}$", model_id))
|
| 109 |
+
|
| 110 |
+
@staticmethod
|
| 111 |
+
def get_api_base_for_model(model: str) -> str:
|
| 112 |
+
"""
|
| 113 |
+
Get the appropriate API base URL for the given model
|
| 114 |
+
"""
|
| 115 |
+
if BasetenConfig.is_dedicated_deployment(model):
|
| 116 |
+
# Extract the model ID (remove 'baseten/' prefix if present)
|
| 117 |
+
model_id = model.replace("baseten/", "")
|
| 118 |
+
return f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1"
|
| 119 |
+
else:
|
| 120 |
+
# Use Model API
|
| 121 |
+
return "https://inference.baseten.co/v1"
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/base_aws_llm.py
ADDED
|
@@ -0,0 +1,1406 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
import urllib.parse
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
from typing import (
|
| 7 |
+
TYPE_CHECKING,
|
| 8 |
+
Any,
|
| 9 |
+
Dict,
|
| 10 |
+
List,
|
| 11 |
+
Literal,
|
| 12 |
+
Optional,
|
| 13 |
+
Tuple,
|
| 14 |
+
Union,
|
| 15 |
+
cast,
|
| 16 |
+
get_args,
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
import httpx
|
| 20 |
+
from pydantic import BaseModel
|
| 21 |
+
|
| 22 |
+
from litellm._logging import verbose_logger
|
| 23 |
+
from litellm.caching.caching import DualCache
|
| 24 |
+
from litellm.constants import (
|
| 25 |
+
BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
|
| 26 |
+
BEDROCK_INVOKE_PROVIDERS_LITERAL,
|
| 27 |
+
BEDROCK_MAX_POLICY_SIZE,
|
| 28 |
+
)
|
| 29 |
+
from litellm.litellm_core_utils.dd_tracing import tracer
|
| 30 |
+
from litellm.secret_managers.main import get_secret, get_secret_str
|
| 31 |
+
|
| 32 |
+
if TYPE_CHECKING:
|
| 33 |
+
from botocore.awsrequest import AWSPreparedRequest
|
| 34 |
+
from botocore.credentials import Credentials
|
| 35 |
+
else:
|
| 36 |
+
Credentials = Any
|
| 37 |
+
AWSPreparedRequest = Any
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class Boto3CredentialsInfo(BaseModel):
|
| 41 |
+
credentials: Credentials
|
| 42 |
+
aws_region_name: str
|
| 43 |
+
aws_bedrock_runtime_endpoint: Optional[str]
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class AwsAuthError(Exception):
|
| 47 |
+
def __init__(self, status_code, message):
|
| 48 |
+
self.status_code = status_code
|
| 49 |
+
self.message = message
|
| 50 |
+
self.request = httpx.Request(
|
| 51 |
+
method="POST", url="https://us-west-2.console.aws.amazon.com/bedrock"
|
| 52 |
+
)
|
| 53 |
+
self.response = httpx.Response(status_code=status_code, request=self.request)
|
| 54 |
+
super().__init__(
|
| 55 |
+
self.message
|
| 56 |
+
) # Call the base class constructor with the parameters it needs
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class BaseAWSLLM:
|
| 60 |
+
def __init__(self) -> None:
|
| 61 |
+
self.iam_cache = DualCache()
|
| 62 |
+
super().__init__()
|
| 63 |
+
self.aws_authentication_params = [
|
| 64 |
+
"aws_access_key_id",
|
| 65 |
+
"aws_secret_access_key",
|
| 66 |
+
"aws_session_token",
|
| 67 |
+
"aws_region_name",
|
| 68 |
+
"aws_session_name",
|
| 69 |
+
"aws_profile_name",
|
| 70 |
+
"aws_role_name",
|
| 71 |
+
"aws_web_identity_token",
|
| 72 |
+
"aws_sts_endpoint",
|
| 73 |
+
"aws_bedrock_runtime_endpoint",
|
| 74 |
+
"aws_external_id",
|
| 75 |
+
]
|
| 76 |
+
|
| 77 |
+
def _get_ssl_verify(self, ssl_verify: Optional[Union[bool, str]] = None):
|
| 78 |
+
"""
|
| 79 |
+
Get SSL verification setting for boto3 clients.
|
| 80 |
+
|
| 81 |
+
This ensures that custom CA certificates are properly used for all AWS API calls,
|
| 82 |
+
including STS and Bedrock services.
|
| 83 |
+
|
| 84 |
+
Returns:
|
| 85 |
+
Union[bool, str]: SSL verification setting - False to disable, True to enable,
|
| 86 |
+
or a string path to a CA bundle file
|
| 87 |
+
"""
|
| 88 |
+
from litellm.llms.custom_httpx.http_handler import get_ssl_verify
|
| 89 |
+
|
| 90 |
+
return get_ssl_verify(ssl_verify=ssl_verify)
|
| 91 |
+
|
| 92 |
+
def get_cache_key(self, credential_args: Dict[str, Optional[str]]) -> str:
|
| 93 |
+
"""
|
| 94 |
+
Generate a unique cache key based on the credential arguments.
|
| 95 |
+
"""
|
| 96 |
+
# Convert credential arguments to a JSON string and hash it to create a unique key
|
| 97 |
+
credential_str = json.dumps(credential_args, sort_keys=True)
|
| 98 |
+
return hashlib.sha256(credential_str.encode()).hexdigest()
|
| 99 |
+
|
| 100 |
+
@tracer.wrap()
|
| 101 |
+
def get_credentials(
|
| 102 |
+
self,
|
| 103 |
+
aws_access_key_id: Optional[str] = None,
|
| 104 |
+
aws_secret_access_key: Optional[str] = None,
|
| 105 |
+
aws_session_token: Optional[str] = None,
|
| 106 |
+
aws_region_name: Optional[str] = None,
|
| 107 |
+
aws_session_name: Optional[str] = None,
|
| 108 |
+
aws_profile_name: Optional[str] = None,
|
| 109 |
+
aws_role_name: Optional[str] = None,
|
| 110 |
+
aws_web_identity_token: Optional[str] = None,
|
| 111 |
+
aws_sts_endpoint: Optional[str] = None,
|
| 112 |
+
aws_external_id: Optional[str] = None,
|
| 113 |
+
ssl_verify: Optional[Union[bool, str]] = None,
|
| 114 |
+
):
|
| 115 |
+
"""
|
| 116 |
+
Return a boto3.Credentials object
|
| 117 |
+
"""
|
| 118 |
+
## CHECK IS 'os.environ/' passed in
|
| 119 |
+
params_to_check: List[Optional[str]] = [
|
| 120 |
+
aws_access_key_id,
|
| 121 |
+
aws_secret_access_key,
|
| 122 |
+
aws_session_token,
|
| 123 |
+
aws_region_name,
|
| 124 |
+
aws_session_name,
|
| 125 |
+
aws_profile_name,
|
| 126 |
+
aws_role_name,
|
| 127 |
+
aws_web_identity_token,
|
| 128 |
+
aws_sts_endpoint,
|
| 129 |
+
aws_external_id,
|
| 130 |
+
]
|
| 131 |
+
|
| 132 |
+
# Iterate over parameters and update if needed
|
| 133 |
+
for i, param in enumerate(params_to_check):
|
| 134 |
+
if param and param.startswith("os.environ/"):
|
| 135 |
+
_v = get_secret(param)
|
| 136 |
+
if _v is not None and isinstance(_v, str):
|
| 137 |
+
params_to_check[i] = _v
|
| 138 |
+
elif param is None: # check if uppercase value in env
|
| 139 |
+
key = self.aws_authentication_params[i]
|
| 140 |
+
if key.upper() in os.environ:
|
| 141 |
+
params_to_check[i] = os.getenv(key.upper())
|
| 142 |
+
|
| 143 |
+
# Assign updated values back to parameters
|
| 144 |
+
(
|
| 145 |
+
aws_access_key_id,
|
| 146 |
+
aws_secret_access_key,
|
| 147 |
+
aws_session_token,
|
| 148 |
+
aws_region_name,
|
| 149 |
+
aws_session_name,
|
| 150 |
+
aws_profile_name,
|
| 151 |
+
aws_role_name,
|
| 152 |
+
aws_web_identity_token,
|
| 153 |
+
aws_sts_endpoint,
|
| 154 |
+
aws_external_id,
|
| 155 |
+
) = params_to_check
|
| 156 |
+
|
| 157 |
+
verbose_logger.debug(
|
| 158 |
+
"in get credentials\n"
|
| 159 |
+
"aws_access_key_id=%s\n"
|
| 160 |
+
"aws_secret_access_key=%s\n"
|
| 161 |
+
"aws_session_token=%s\n"
|
| 162 |
+
"aws_region_name=%s\n"
|
| 163 |
+
"aws_session_name=%s\n"
|
| 164 |
+
"aws_profile_name=%s\n"
|
| 165 |
+
"aws_role_name=%s\n"
|
| 166 |
+
"aws_web_identity_token=%s\n"
|
| 167 |
+
"aws_sts_endpoint=%s\n"
|
| 168 |
+
"aws_external_id=%s",
|
| 169 |
+
aws_access_key_id,
|
| 170 |
+
aws_secret_access_key,
|
| 171 |
+
aws_session_token,
|
| 172 |
+
aws_region_name,
|
| 173 |
+
aws_session_name,
|
| 174 |
+
aws_profile_name,
|
| 175 |
+
aws_role_name,
|
| 176 |
+
aws_web_identity_token,
|
| 177 |
+
aws_sts_endpoint,
|
| 178 |
+
aws_external_id,
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
# create cache key for non-expiring auth flows
|
| 182 |
+
args = {
|
| 183 |
+
k: v
|
| 184 |
+
for k, v in locals().items()
|
| 185 |
+
if k.startswith("aws_") or k == "ssl_verify"
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
cache_key = self.get_cache_key(args)
|
| 189 |
+
_cached_credentials = self.iam_cache.get_cache(cache_key)
|
| 190 |
+
if _cached_credentials:
|
| 191 |
+
return _cached_credentials
|
| 192 |
+
|
| 193 |
+
#########################################################
|
| 194 |
+
# Handle diff boto3 auth flows
|
| 195 |
+
# for each helper
|
| 196 |
+
# Return:
|
| 197 |
+
# Credentials - boto3.Credentials
|
| 198 |
+
# cache ttl - Optional[int]. If None, the credentials are not cached. Some auth flows have no expiry time.
|
| 199 |
+
#########################################################
|
| 200 |
+
if (
|
| 201 |
+
aws_web_identity_token is not None
|
| 202 |
+
and aws_role_name is not None
|
| 203 |
+
and aws_session_name is not None
|
| 204 |
+
):
|
| 205 |
+
credentials, _cache_ttl = self._auth_with_web_identity_token(
|
| 206 |
+
aws_web_identity_token=aws_web_identity_token,
|
| 207 |
+
aws_role_name=aws_role_name,
|
| 208 |
+
aws_session_name=aws_session_name,
|
| 209 |
+
aws_region_name=aws_region_name,
|
| 210 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 211 |
+
aws_external_id=aws_external_id,
|
| 212 |
+
)
|
| 213 |
+
elif aws_role_name is not None:
|
| 214 |
+
# Check if we're already running as the target role and can skip assumption
|
| 215 |
+
# This handles IRSA (EKS), ECS task roles, and EC2 instance profiles
|
| 216 |
+
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
|
| 217 |
+
verbose_logger.debug(
|
| 218 |
+
"Already running as target role %s, using ambient credentials",
|
| 219 |
+
aws_role_name,
|
| 220 |
+
)
|
| 221 |
+
credentials, _cache_ttl = self._auth_with_env_vars()
|
| 222 |
+
else:
|
| 223 |
+
verbose_logger.debug(
|
| 224 |
+
"Using role assumption: calling _auth_with_aws_role"
|
| 225 |
+
)
|
| 226 |
+
# If aws_session_name is not provided, generate a default one
|
| 227 |
+
if aws_session_name is None:
|
| 228 |
+
aws_session_name = (
|
| 229 |
+
f"litellm-session-{int(datetime.now().timestamp())}"
|
| 230 |
+
)
|
| 231 |
+
credentials, _cache_ttl = self._auth_with_aws_role(
|
| 232 |
+
aws_access_key_id=aws_access_key_id,
|
| 233 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 234 |
+
aws_session_token=aws_session_token,
|
| 235 |
+
aws_role_name=aws_role_name,
|
| 236 |
+
aws_session_name=aws_session_name,
|
| 237 |
+
aws_region_name=aws_region_name,
|
| 238 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 239 |
+
aws_external_id=aws_external_id,
|
| 240 |
+
ssl_verify=ssl_verify,
|
| 241 |
+
)
|
| 242 |
+
|
| 243 |
+
elif aws_profile_name is not None: ### CHECK SESSION ###
|
| 244 |
+
credentials, _cache_ttl = self._auth_with_aws_profile(aws_profile_name)
|
| 245 |
+
elif (
|
| 246 |
+
aws_access_key_id is not None
|
| 247 |
+
and aws_secret_access_key is not None
|
| 248 |
+
and aws_session_token is not None
|
| 249 |
+
):
|
| 250 |
+
credentials, _cache_ttl = self._auth_with_aws_session_token(
|
| 251 |
+
aws_access_key_id=aws_access_key_id,
|
| 252 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 253 |
+
aws_session_token=aws_session_token,
|
| 254 |
+
)
|
| 255 |
+
elif (
|
| 256 |
+
aws_access_key_id is not None
|
| 257 |
+
and aws_secret_access_key is not None
|
| 258 |
+
and aws_region_name is not None
|
| 259 |
+
):
|
| 260 |
+
credentials, _cache_ttl = self._auth_with_access_key_and_secret_key(
|
| 261 |
+
aws_access_key_id=aws_access_key_id,
|
| 262 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 263 |
+
aws_region_name=aws_region_name,
|
| 264 |
+
)
|
| 265 |
+
else:
|
| 266 |
+
credentials, _cache_ttl = self._auth_with_env_vars()
|
| 267 |
+
|
| 268 |
+
self.iam_cache.set_cache(cache_key, credentials, ttl=_cache_ttl)
|
| 269 |
+
return credentials
|
| 270 |
+
|
| 271 |
+
def _get_aws_region_from_model_arn(self, model: Optional[str]) -> Optional[str]:
|
| 272 |
+
try:
|
| 273 |
+
# First check if the string contains the expected prefix
|
| 274 |
+
if not isinstance(model, str) or "arn:aws:bedrock" not in model:
|
| 275 |
+
return None
|
| 276 |
+
|
| 277 |
+
# Split the ARN and check if we have enough parts
|
| 278 |
+
parts = model.split(":")
|
| 279 |
+
if len(parts) < 4:
|
| 280 |
+
return None
|
| 281 |
+
|
| 282 |
+
# Get the region from the correct position
|
| 283 |
+
region = parts[3]
|
| 284 |
+
if not region: # Check if region is empty
|
| 285 |
+
return None
|
| 286 |
+
|
| 287 |
+
return region
|
| 288 |
+
except Exception:
|
| 289 |
+
# Catch any unexpected errors and return None
|
| 290 |
+
return None
|
| 291 |
+
|
| 292 |
+
@staticmethod
|
| 293 |
+
def _get_provider_from_model_path(
|
| 294 |
+
model_path: str,
|
| 295 |
+
) -> Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL]:
|
| 296 |
+
"""
|
| 297 |
+
Helper function to get the provider from a model path with format: provider/model-name
|
| 298 |
+
|
| 299 |
+
Args:
|
| 300 |
+
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
|
| 301 |
+
|
| 302 |
+
Returns:
|
| 303 |
+
Optional[str]: The provider name, or None if no valid provider found
|
| 304 |
+
"""
|
| 305 |
+
parts = model_path.split("/")
|
| 306 |
+
if len(parts) >= 1:
|
| 307 |
+
provider = parts[0]
|
| 308 |
+
if provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 309 |
+
return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
|
| 310 |
+
return None
|
| 311 |
+
|
| 312 |
+
@staticmethod
|
| 313 |
+
def get_bedrock_invoke_provider(
|
| 314 |
+
model: str,
|
| 315 |
+
) -> Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL]:
|
| 316 |
+
"""
|
| 317 |
+
Helper function to get the bedrock provider from the model
|
| 318 |
+
|
| 319 |
+
handles 3 scenarions:
|
| 320 |
+
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
| 321 |
+
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
| 322 |
+
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
|
| 323 |
+
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
|
| 324 |
+
"""
|
| 325 |
+
if model.startswith("invoke/"):
|
| 326 |
+
model = model.replace("invoke/", "", 1)
|
| 327 |
+
|
| 328 |
+
# Special case: Check for "nova" in model name first (before "amazon")
|
| 329 |
+
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
|
| 330 |
+
if "nova" in model.lower():
|
| 331 |
+
if "nova" in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 332 |
+
return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
|
| 333 |
+
|
| 334 |
+
_split_model = model.split(".")[0]
|
| 335 |
+
if _split_model in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 336 |
+
return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
|
| 337 |
+
|
| 338 |
+
# If not a known provider, check for pattern with two slashes
|
| 339 |
+
provider = BaseAWSLLM._get_provider_from_model_path(model)
|
| 340 |
+
if provider is not None:
|
| 341 |
+
return provider
|
| 342 |
+
|
| 343 |
+
for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 344 |
+
if provider in model:
|
| 345 |
+
return provider
|
| 346 |
+
return None
|
| 347 |
+
|
| 348 |
+
@staticmethod
|
| 349 |
+
def get_bedrock_model_id(
|
| 350 |
+
optional_params: dict,
|
| 351 |
+
provider: Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL],
|
| 352 |
+
model: str,
|
| 353 |
+
) -> str:
|
| 354 |
+
model_id = optional_params.pop("model_id", None)
|
| 355 |
+
if model_id is not None:
|
| 356 |
+
model_id = BaseAWSLLM.encode_model_id(model_id=model_id)
|
| 357 |
+
else:
|
| 358 |
+
model_id = model
|
| 359 |
+
|
| 360 |
+
model_id = model_id.replace("invoke/", "", 1)
|
| 361 |
+
if provider == "llama" and "llama/" in model_id:
|
| 362 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 363 |
+
model_id, spec="llama"
|
| 364 |
+
)
|
| 365 |
+
elif provider == "deepseek_r1" and "deepseek_r1/" in model_id:
|
| 366 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 367 |
+
model_id, spec="deepseek_r1"
|
| 368 |
+
)
|
| 369 |
+
elif provider == "openai" and "openai/" in model_id:
|
| 370 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 371 |
+
model_id, spec="openai"
|
| 372 |
+
)
|
| 373 |
+
elif provider == "qwen2" and "qwen2/" in model_id:
|
| 374 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 375 |
+
model_id, spec="qwen2"
|
| 376 |
+
)
|
| 377 |
+
elif provider == "qwen3" and "qwen3/" in model_id:
|
| 378 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 379 |
+
model_id, spec="qwen3"
|
| 380 |
+
)
|
| 381 |
+
elif provider == "stability" and "stability/" in model_id:
|
| 382 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 383 |
+
model_id, spec="stability"
|
| 384 |
+
)
|
| 385 |
+
elif provider == "moonshot" and "moonshot/" in model_id:
|
| 386 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 387 |
+
model_id, spec="moonshot"
|
| 388 |
+
)
|
| 389 |
+
elif "nova-2/" in model_id:
|
| 390 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 391 |
+
model_id, spec="nova-2"
|
| 392 |
+
)
|
| 393 |
+
elif "nova/" in model_id:
|
| 394 |
+
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
|
| 395 |
+
model_id, spec="nova"
|
| 396 |
+
)
|
| 397 |
+
return model_id
|
| 398 |
+
|
| 399 |
+
@staticmethod
|
| 400 |
+
def _get_model_id_from_model_with_spec(
|
| 401 |
+
model: str,
|
| 402 |
+
spec: str,
|
| 403 |
+
) -> str:
|
| 404 |
+
"""
|
| 405 |
+
Remove `llama` from modelID since `llama` is simply a spec to follow for custom bedrock models
|
| 406 |
+
"""
|
| 407 |
+
model_id = model.replace(spec + "/", "")
|
| 408 |
+
return BaseAWSLLM.encode_model_id(model_id=model_id)
|
| 409 |
+
|
| 410 |
+
@staticmethod
|
| 411 |
+
def encode_model_id(model_id: str) -> str:
|
| 412 |
+
"""
|
| 413 |
+
Double encode the model ID to ensure it matches the expected double-encoded format.
|
| 414 |
+
Args:
|
| 415 |
+
model_id (str): The model ID to encode.
|
| 416 |
+
Returns:
|
| 417 |
+
str: The double-encoded model ID.
|
| 418 |
+
"""
|
| 419 |
+
return urllib.parse.quote(model_id, safe="")
|
| 420 |
+
|
| 421 |
+
@staticmethod
|
| 422 |
+
def get_bedrock_embedding_provider(
|
| 423 |
+
model: str,
|
| 424 |
+
) -> Optional[BEDROCK_EMBEDDING_PROVIDERS_LITERAL]:
|
| 425 |
+
"""
|
| 426 |
+
Helper function to get the bedrock embedding provider from the model
|
| 427 |
+
|
| 428 |
+
Handles scenarios like:
|
| 429 |
+
1. model=cohere.embed-english-v3:0 -> Returns `cohere`
|
| 430 |
+
2. model=amazon.titan-embed-text-v1 -> Returns `amazon`
|
| 431 |
+
3. model=amazon.nova-2-multimodal-embeddings-v1:0 -> Returns `nova`
|
| 432 |
+
4. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
|
| 433 |
+
5. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
|
| 434 |
+
"""
|
| 435 |
+
# Special case: Check for "nova" in model name first (before "amazon")
|
| 436 |
+
# This handles amazon.nova-* models
|
| 437 |
+
if "nova" in model.lower():
|
| 438 |
+
if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
|
| 439 |
+
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova")
|
| 440 |
+
|
| 441 |
+
# Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0
|
| 442 |
+
if "." in model:
|
| 443 |
+
parts = model.split(".")
|
| 444 |
+
# Check if the second part (after potential region) is a known provider
|
| 445 |
+
if len(parts) >= 2:
|
| 446 |
+
potential_provider = parts[
|
| 447 |
+
1
|
| 448 |
+
] # e.g., "twelvelabs" from "us.twelvelabs.marengo-embed-2-7-v1:0"
|
| 449 |
+
if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
|
| 450 |
+
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider)
|
| 451 |
+
|
| 452 |
+
# Check if the first part is a known provider (standard format)
|
| 453 |
+
potential_provider = parts[
|
| 454 |
+
0
|
| 455 |
+
] # e.g., "cohere" from "cohere.embed-english-v3:0"
|
| 456 |
+
if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
|
| 457 |
+
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider)
|
| 458 |
+
|
| 459 |
+
# Fallback: check if any provider name appears in the model string
|
| 460 |
+
for provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
|
| 461 |
+
if provider in model:
|
| 462 |
+
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, provider)
|
| 463 |
+
|
| 464 |
+
return None
|
| 465 |
+
|
| 466 |
+
def _get_aws_region_name(
|
| 467 |
+
self,
|
| 468 |
+
optional_params: dict,
|
| 469 |
+
model: Optional[str] = None,
|
| 470 |
+
model_id: Optional[str] = None,
|
| 471 |
+
) -> str:
|
| 472 |
+
"""
|
| 473 |
+
Get the AWS region name from the environment variables.
|
| 474 |
+
|
| 475 |
+
Parameters:
|
| 476 |
+
optional_params (dict): Optional parameters for the model call
|
| 477 |
+
model (str): The model name
|
| 478 |
+
model_id (str): The model ID. This is the ARN of the model, if passed in as a separate param.
|
| 479 |
+
|
| 480 |
+
Returns:
|
| 481 |
+
str: The AWS region name
|
| 482 |
+
"""
|
| 483 |
+
aws_region_name = optional_params.get("aws_region_name", None)
|
| 484 |
+
### SET REGION NAME ###
|
| 485 |
+
if aws_region_name is None:
|
| 486 |
+
# check model arn #
|
| 487 |
+
if model_id is not None:
|
| 488 |
+
aws_region_name = self._get_aws_region_from_model_arn(model_id)
|
| 489 |
+
else:
|
| 490 |
+
aws_region_name = self._get_aws_region_from_model_arn(model)
|
| 491 |
+
# check env #
|
| 492 |
+
litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
|
| 493 |
+
|
| 494 |
+
if (
|
| 495 |
+
aws_region_name is None
|
| 496 |
+
and litellm_aws_region_name is not None
|
| 497 |
+
and isinstance(litellm_aws_region_name, str)
|
| 498 |
+
):
|
| 499 |
+
aws_region_name = litellm_aws_region_name
|
| 500 |
+
|
| 501 |
+
standard_aws_region_name = get_secret("AWS_REGION", None)
|
| 502 |
+
if (
|
| 503 |
+
aws_region_name is None
|
| 504 |
+
and standard_aws_region_name is not None
|
| 505 |
+
and isinstance(standard_aws_region_name, str)
|
| 506 |
+
):
|
| 507 |
+
aws_region_name = standard_aws_region_name
|
| 508 |
+
if aws_region_name is None:
|
| 509 |
+
try:
|
| 510 |
+
import boto3
|
| 511 |
+
|
| 512 |
+
with tracer.trace("boto3.Session()"):
|
| 513 |
+
session = boto3.Session()
|
| 514 |
+
configured_region = session.region_name
|
| 515 |
+
if configured_region:
|
| 516 |
+
aws_region_name = configured_region
|
| 517 |
+
else:
|
| 518 |
+
aws_region_name = "us-west-2"
|
| 519 |
+
except Exception:
|
| 520 |
+
aws_region_name = "us-west-2"
|
| 521 |
+
|
| 522 |
+
return aws_region_name
|
| 523 |
+
|
| 524 |
+
def get_aws_region_name_for_non_llm_api_calls(
|
| 525 |
+
self,
|
| 526 |
+
aws_region_name: Optional[str] = None,
|
| 527 |
+
):
|
| 528 |
+
"""
|
| 529 |
+
Get the AWS region name for non-llm api calls.
|
| 530 |
+
|
| 531 |
+
LLM API calls check the model arn and end up using that as the region name.
|
| 532 |
+
|
| 533 |
+
For non-llm api calls eg. Guardrails, Vector Stores we just need to check the dynamic param or env vars.
|
| 534 |
+
"""
|
| 535 |
+
if aws_region_name is None:
|
| 536 |
+
# check env #
|
| 537 |
+
litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
|
| 538 |
+
|
| 539 |
+
if litellm_aws_region_name is not None and isinstance(
|
| 540 |
+
litellm_aws_region_name, str
|
| 541 |
+
):
|
| 542 |
+
aws_region_name = litellm_aws_region_name
|
| 543 |
+
|
| 544 |
+
standard_aws_region_name = get_secret("AWS_REGION", None)
|
| 545 |
+
if standard_aws_region_name is not None and isinstance(
|
| 546 |
+
standard_aws_region_name, str
|
| 547 |
+
):
|
| 548 |
+
aws_region_name = standard_aws_region_name
|
| 549 |
+
|
| 550 |
+
if aws_region_name is None:
|
| 551 |
+
aws_region_name = "us-west-2"
|
| 552 |
+
return aws_region_name
|
| 553 |
+
|
| 554 |
+
@staticmethod
|
| 555 |
+
def _parse_arn_account_and_role_name(
|
| 556 |
+
arn: str,
|
| 557 |
+
) -> Optional[Tuple[str, str, str]]:
|
| 558 |
+
"""
|
| 559 |
+
Parse an ARN and return (partition, account_id, role_name).
|
| 560 |
+
|
| 561 |
+
Handles:
|
| 562 |
+
- arn:aws:iam::123456789012:role/MyRole
|
| 563 |
+
- arn:aws:iam::123456789012:role/path/to/MyRole
|
| 564 |
+
- arn:aws:sts::123456789012:assumed-role/MyRole/session-name
|
| 565 |
+
|
| 566 |
+
Returns None if the ARN cannot be parsed.
|
| 567 |
+
"""
|
| 568 |
+
# ARN format: arn:PARTITION:SERVICE:REGION:ACCOUNT:RESOURCE
|
| 569 |
+
parts = arn.split(":")
|
| 570 |
+
if len(parts) < 6 or parts[0] != "arn":
|
| 571 |
+
return None
|
| 572 |
+
|
| 573 |
+
partition = parts[1] # e.g. "aws", "aws-cn", "aws-us-gov"
|
| 574 |
+
account_id = parts[4]
|
| 575 |
+
resource = ":".join(parts[5:]) # rejoin in case resource contains colons
|
| 576 |
+
|
| 577 |
+
if resource.startswith("role/"):
|
| 578 |
+
# arn:aws:iam::ACCOUNT:role/[path/]ROLE_NAME
|
| 579 |
+
role_name = resource.split("/")[-1]
|
| 580 |
+
elif resource.startswith("assumed-role/"):
|
| 581 |
+
# arn:aws:sts::ACCOUNT:assumed-role/ROLE_NAME/SESSION
|
| 582 |
+
role_parts = resource.split("/")
|
| 583 |
+
if len(role_parts) >= 2:
|
| 584 |
+
role_name = role_parts[1]
|
| 585 |
+
else:
|
| 586 |
+
return None
|
| 587 |
+
else:
|
| 588 |
+
return None
|
| 589 |
+
|
| 590 |
+
return partition, account_id, role_name
|
| 591 |
+
|
| 592 |
+
def _is_already_running_as_role(
|
| 593 |
+
self,
|
| 594 |
+
aws_role_name: str,
|
| 595 |
+
ssl_verify: Optional[Union[bool, str]] = None,
|
| 596 |
+
) -> bool:
|
| 597 |
+
"""
|
| 598 |
+
Check if the current environment is already running as the target IAM role.
|
| 599 |
+
|
| 600 |
+
This handles multiple AWS environments:
|
| 601 |
+
- IRSA (EKS): AWS_ROLE_ARN + AWS_WEB_IDENTITY_TOKEN_FILE are set
|
| 602 |
+
- ECS task roles: Uses sts:GetCallerIdentity to check current role ARN
|
| 603 |
+
- EC2 instance profiles: Uses sts:GetCallerIdentity to check current role ARN
|
| 604 |
+
|
| 605 |
+
Compares partition, account ID, and role name to avoid cross-account
|
| 606 |
+
false matches.
|
| 607 |
+
|
| 608 |
+
Returns True if the current identity matches the target role, meaning
|
| 609 |
+
we can skip sts:AssumeRole and use ambient credentials directly.
|
| 610 |
+
"""
|
| 611 |
+
target_parsed = self._parse_arn_account_and_role_name(aws_role_name)
|
| 612 |
+
if target_parsed is None:
|
| 613 |
+
return False
|
| 614 |
+
|
| 615 |
+
target_partition, target_account, target_role = target_parsed
|
| 616 |
+
|
| 617 |
+
# Fast path: IRSA environment check (no API call needed)
|
| 618 |
+
current_role_arn = os.getenv("AWS_ROLE_ARN")
|
| 619 |
+
web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
|
| 620 |
+
if current_role_arn and web_identity_token_file:
|
| 621 |
+
return current_role_arn == aws_role_name
|
| 622 |
+
|
| 623 |
+
# For ECS/EC2: call sts:GetCallerIdentity to check if already running as the role
|
| 624 |
+
try:
|
| 625 |
+
import boto3
|
| 626 |
+
|
| 627 |
+
with tracer.trace("boto3.client(sts).get_caller_identity"):
|
| 628 |
+
sts_client = boto3.client(
|
| 629 |
+
"sts", verify=self._get_ssl_verify(ssl_verify)
|
| 630 |
+
)
|
| 631 |
+
identity = sts_client.get_caller_identity()
|
| 632 |
+
caller_arn = identity.get("Arn", "")
|
| 633 |
+
|
| 634 |
+
caller_parsed = self._parse_arn_account_and_role_name(caller_arn)
|
| 635 |
+
if caller_parsed is not None:
|
| 636 |
+
caller_partition, caller_account, caller_role = caller_parsed
|
| 637 |
+
if (
|
| 638 |
+
caller_partition == target_partition
|
| 639 |
+
and caller_account == target_account
|
| 640 |
+
and caller_role == target_role
|
| 641 |
+
):
|
| 642 |
+
verbose_logger.debug(
|
| 643 |
+
"Current identity already matches target role: %s",
|
| 644 |
+
aws_role_name,
|
| 645 |
+
)
|
| 646 |
+
return True
|
| 647 |
+
|
| 648 |
+
except Exception as e:
|
| 649 |
+
verbose_logger.debug(
|
| 650 |
+
"Could not determine current role identity: %s", str(e)
|
| 651 |
+
)
|
| 652 |
+
|
| 653 |
+
return False
|
| 654 |
+
|
| 655 |
+
@tracer.wrap()
|
| 656 |
+
def _auth_with_web_identity_token(
|
| 657 |
+
self,
|
| 658 |
+
aws_web_identity_token: str,
|
| 659 |
+
aws_role_name: str,
|
| 660 |
+
aws_session_name: str,
|
| 661 |
+
aws_region_name: Optional[str],
|
| 662 |
+
aws_sts_endpoint: Optional[str],
|
| 663 |
+
aws_external_id: Optional[str] = None,
|
| 664 |
+
ssl_verify: Optional[Union[bool, str]] = None,
|
| 665 |
+
) -> Tuple[Credentials, Optional[int]]:
|
| 666 |
+
"""
|
| 667 |
+
Authenticate with AWS Web Identity Token
|
| 668 |
+
"""
|
| 669 |
+
import boto3
|
| 670 |
+
|
| 671 |
+
verbose_logger.debug(
|
| 672 |
+
f"IN Web Identity Token: {aws_web_identity_token} | Role Name: {aws_role_name} | Session Name: {aws_session_name}"
|
| 673 |
+
)
|
| 674 |
+
|
| 675 |
+
if aws_sts_endpoint is None:
|
| 676 |
+
sts_endpoint = f"https://sts.{aws_region_name}.amazonaws.com"
|
| 677 |
+
else:
|
| 678 |
+
sts_endpoint = aws_sts_endpoint
|
| 679 |
+
|
| 680 |
+
oidc_token = get_secret(aws_web_identity_token)
|
| 681 |
+
|
| 682 |
+
if oidc_token is None:
|
| 683 |
+
raise AwsAuthError(
|
| 684 |
+
message="OIDC token could not be retrieved from secret manager.",
|
| 685 |
+
status_code=401,
|
| 686 |
+
)
|
| 687 |
+
|
| 688 |
+
with tracer.trace("boto3.client(sts)"):
|
| 689 |
+
sts_client = boto3.client(
|
| 690 |
+
"sts",
|
| 691 |
+
region_name=aws_region_name,
|
| 692 |
+
endpoint_url=sts_endpoint,
|
| 693 |
+
verify=self._get_ssl_verify(ssl_verify),
|
| 694 |
+
)
|
| 695 |
+
|
| 696 |
+
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
|
| 697 |
+
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
|
| 698 |
+
assume_role_params = {
|
| 699 |
+
"RoleArn": aws_role_name,
|
| 700 |
+
"RoleSessionName": aws_session_name,
|
| 701 |
+
"WebIdentityToken": oidc_token,
|
| 702 |
+
"DurationSeconds": 3600,
|
| 703 |
+
"Policy": '{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringLike":{"aws:UserAgent":"litellm/*"}}}]}',
|
| 704 |
+
}
|
| 705 |
+
|
| 706 |
+
# Add ExternalId parameter if provided
|
| 707 |
+
if aws_external_id is not None:
|
| 708 |
+
assume_role_params["ExternalId"] = aws_external_id
|
| 709 |
+
|
| 710 |
+
sts_response = sts_client.assume_role_with_web_identity(**assume_role_params)
|
| 711 |
+
|
| 712 |
+
iam_creds_dict = {
|
| 713 |
+
"aws_access_key_id": sts_response["Credentials"]["AccessKeyId"],
|
| 714 |
+
"aws_secret_access_key": sts_response["Credentials"]["SecretAccessKey"],
|
| 715 |
+
"aws_session_token": sts_response["Credentials"]["SessionToken"],
|
| 716 |
+
"region_name": aws_region_name,
|
| 717 |
+
}
|
| 718 |
+
|
| 719 |
+
if sts_response["PackedPolicySize"] > BEDROCK_MAX_POLICY_SIZE:
|
| 720 |
+
verbose_logger.warning(
|
| 721 |
+
f"The policy size is greater than 75% of the allowed size, PackedPolicySize: {sts_response['PackedPolicySize']}"
|
| 722 |
+
)
|
| 723 |
+
|
| 724 |
+
with tracer.trace("boto3.Session(**iam_creds_dict)"):
|
| 725 |
+
session = boto3.Session(**iam_creds_dict)
|
| 726 |
+
|
| 727 |
+
iam_creds = session.get_credentials()
|
| 728 |
+
return iam_creds, self._get_default_ttl_for_boto3_credentials()
|
| 729 |
+
|
| 730 |
+
def _handle_irsa_cross_account(
|
| 731 |
+
self,
|
| 732 |
+
irsa_role_arn: str,
|
| 733 |
+
aws_role_name: str,
|
| 734 |
+
aws_session_name: str,
|
| 735 |
+
region: str,
|
| 736 |
+
web_identity_token_file: str,
|
| 737 |
+
aws_external_id: Optional[str] = None,
|
| 738 |
+
aws_sts_endpoint: Optional[str] = None,
|
| 739 |
+
ssl_verify: Optional[Union[bool, str]] = None,
|
| 740 |
+
) -> dict:
|
| 741 |
+
"""Handle cross-account role assumption for IRSA."""
|
| 742 |
+
import boto3
|
| 743 |
+
|
| 744 |
+
verbose_logger.debug("Cross-account role assumption detected")
|
| 745 |
+
|
| 746 |
+
# Read the web identity token
|
| 747 |
+
with open(web_identity_token_file, "r") as f:
|
| 748 |
+
web_identity_token = f.read().strip()
|
| 749 |
+
|
| 750 |
+
irsa_sts_kwargs: dict = {
|
| 751 |
+
"region_name": region,
|
| 752 |
+
"verify": self._get_ssl_verify(ssl_verify),
|
| 753 |
+
}
|
| 754 |
+
if aws_sts_endpoint is not None:
|
| 755 |
+
irsa_sts_kwargs["endpoint_url"] = aws_sts_endpoint
|
| 756 |
+
|
| 757 |
+
# Create an STS client without credentials
|
| 758 |
+
with tracer.trace("boto3.client(sts) for manual IRSA"):
|
| 759 |
+
sts_client = boto3.client("sts", **irsa_sts_kwargs)
|
| 760 |
+
|
| 761 |
+
# Manually assume the IRSA role with the session name
|
| 762 |
+
verbose_logger.debug(
|
| 763 |
+
f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}"
|
| 764 |
+
)
|
| 765 |
+
irsa_response = sts_client.assume_role_with_web_identity(
|
| 766 |
+
RoleArn=irsa_role_arn,
|
| 767 |
+
RoleSessionName=aws_session_name,
|
| 768 |
+
WebIdentityToken=web_identity_token,
|
| 769 |
+
)
|
| 770 |
+
|
| 771 |
+
# Extract the credentials from the IRSA assumption
|
| 772 |
+
irsa_creds = irsa_response["Credentials"]
|
| 773 |
+
|
| 774 |
+
# Create a new STS client with the IRSA credentials
|
| 775 |
+
with tracer.trace("boto3.client(sts) with manual IRSA credentials"):
|
| 776 |
+
sts_client_with_creds = boto3.client(
|
| 777 |
+
"sts",
|
| 778 |
+
aws_access_key_id=irsa_creds["AccessKeyId"],
|
| 779 |
+
aws_secret_access_key=irsa_creds["SecretAccessKey"],
|
| 780 |
+
aws_session_token=irsa_creds["SessionToken"],
|
| 781 |
+
**irsa_sts_kwargs,
|
| 782 |
+
)
|
| 783 |
+
|
| 784 |
+
# Get current caller identity for debugging
|
| 785 |
+
try:
|
| 786 |
+
caller_identity = sts_client_with_creds.get_caller_identity()
|
| 787 |
+
verbose_logger.debug(
|
| 788 |
+
f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}"
|
| 789 |
+
)
|
| 790 |
+
except Exception as e:
|
| 791 |
+
verbose_logger.debug(f"Failed to get caller identity: {e}")
|
| 792 |
+
|
| 793 |
+
# Now assume the target role
|
| 794 |
+
verbose_logger.debug(
|
| 795 |
+
f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}"
|
| 796 |
+
)
|
| 797 |
+
assume_role_params = {
|
| 798 |
+
"RoleArn": aws_role_name,
|
| 799 |
+
"RoleSessionName": aws_session_name,
|
| 800 |
+
}
|
| 801 |
+
|
| 802 |
+
# Add ExternalId parameter if provided
|
| 803 |
+
if aws_external_id is not None:
|
| 804 |
+
assume_role_params["ExternalId"] = aws_external_id
|
| 805 |
+
|
| 806 |
+
return sts_client_with_creds.assume_role(**assume_role_params)
|
| 807 |
+
|
| 808 |
+
def _handle_irsa_same_account(
|
| 809 |
+
self,
|
| 810 |
+
aws_role_name: str,
|
| 811 |
+
aws_session_name: str,
|
| 812 |
+
region: str,
|
| 813 |
+
aws_external_id: Optional[str] = None,
|
| 814 |
+
aws_sts_endpoint: Optional[str] = None,
|
| 815 |
+
ssl_verify: Optional[Union[bool, str]] = None,
|
| 816 |
+
) -> dict:
|
| 817 |
+
"""Handle same-account role assumption for IRSA."""
|
| 818 |
+
import boto3
|
| 819 |
+
|
| 820 |
+
irsa_sts_kwargs: dict = {
|
| 821 |
+
"region_name": region,
|
| 822 |
+
"verify": self._get_ssl_verify(ssl_verify),
|
| 823 |
+
}
|
| 824 |
+
if aws_sts_endpoint is not None:
|
| 825 |
+
irsa_sts_kwargs["endpoint_url"] = aws_sts_endpoint
|
| 826 |
+
|
| 827 |
+
verbose_logger.debug("Same account role assumption, using automatic IRSA")
|
| 828 |
+
with tracer.trace("boto3.client(sts) with automatic IRSA"):
|
| 829 |
+
sts_client = boto3.client("sts", **irsa_sts_kwargs)
|
| 830 |
+
|
| 831 |
+
# Get current caller identity for debugging
|
| 832 |
+
try:
|
| 833 |
+
caller_identity = sts_client.get_caller_identity()
|
| 834 |
+
verbose_logger.debug(
|
| 835 |
+
f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}"
|
| 836 |
+
)
|
| 837 |
+
except Exception as e:
|
| 838 |
+
verbose_logger.debug(f"Failed to get caller identity: {e}")
|
| 839 |
+
|
| 840 |
+
# Assume the role
|
| 841 |
+
verbose_logger.debug(
|
| 842 |
+
f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}"
|
| 843 |
+
)
|
| 844 |
+
assume_role_params = {
|
| 845 |
+
"RoleArn": aws_role_name,
|
| 846 |
+
"RoleSessionName": aws_session_name,
|
| 847 |
+
}
|
| 848 |
+
|
| 849 |
+
# Add ExternalId parameter if provided
|
| 850 |
+
if aws_external_id is not None:
|
| 851 |
+
assume_role_params["ExternalId"] = aws_external_id
|
| 852 |
+
|
| 853 |
+
return sts_client.assume_role(**assume_role_params)
|
| 854 |
+
|
| 855 |
+
def _extract_credentials_and_ttl(
|
| 856 |
+
self, sts_response: dict
|
| 857 |
+
) -> Tuple[Credentials, Optional[int]]:
|
| 858 |
+
"""Extract credentials and TTL from STS response."""
|
| 859 |
+
from botocore.credentials import Credentials
|
| 860 |
+
|
| 861 |
+
sts_credentials = sts_response["Credentials"]
|
| 862 |
+
credentials = Credentials(
|
| 863 |
+
access_key=sts_credentials["AccessKeyId"],
|
| 864 |
+
secret_key=sts_credentials["SecretAccessKey"],
|
| 865 |
+
token=sts_credentials["SessionToken"],
|
| 866 |
+
)
|
| 867 |
+
|
| 868 |
+
expiration_time = sts_credentials["Expiration"]
|
| 869 |
+
ttl = int(
|
| 870 |
+
(expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds()
|
| 871 |
+
)
|
| 872 |
+
|
| 873 |
+
return credentials, ttl
|
| 874 |
+
|
| 875 |
+
@tracer.wrap()
|
| 876 |
+
def _auth_with_aws_role(
|
| 877 |
+
self,
|
| 878 |
+
aws_access_key_id: Optional[str],
|
| 879 |
+
aws_secret_access_key: Optional[str],
|
| 880 |
+
aws_session_token: Optional[str],
|
| 881 |
+
aws_role_name: str,
|
| 882 |
+
aws_session_name: str,
|
| 883 |
+
aws_region_name: Optional[str] = None,
|
| 884 |
+
aws_sts_endpoint: Optional[str] = None,
|
| 885 |
+
aws_external_id: Optional[str] = None,
|
| 886 |
+
ssl_verify: Optional[Union[bool, str]] = None,
|
| 887 |
+
) -> Tuple[Credentials, Optional[int]]:
|
| 888 |
+
"""
|
| 889 |
+
Authenticate with AWS Role
|
| 890 |
+
"""
|
| 891 |
+
import boto3
|
| 892 |
+
from botocore.credentials import Credentials
|
| 893 |
+
|
| 894 |
+
# Check if we're in an EKS/IRSA environment
|
| 895 |
+
web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
|
| 896 |
+
irsa_role_arn = os.getenv("AWS_ROLE_ARN")
|
| 897 |
+
|
| 898 |
+
region = (
|
| 899 |
+
aws_region_name
|
| 900 |
+
or os.getenv("AWS_REGION")
|
| 901 |
+
or os.getenv("AWS_DEFAULT_REGION")
|
| 902 |
+
)
|
| 903 |
+
|
| 904 |
+
# If we have IRSA environment variables and no explicit credentials,
|
| 905 |
+
# we need to use the web identity token flow
|
| 906 |
+
if (
|
| 907 |
+
web_identity_token_file
|
| 908 |
+
and irsa_role_arn
|
| 909 |
+
and aws_access_key_id is None
|
| 910 |
+
and aws_secret_access_key is None
|
| 911 |
+
):
|
| 912 |
+
# For cross-account role assumption with specific session names,
|
| 913 |
+
# we need to manually assume the IRSA role first with the correct session name
|
| 914 |
+
verbose_logger.debug(
|
| 915 |
+
f"IRSA detected: using web identity token from {web_identity_token_file}"
|
| 916 |
+
)
|
| 917 |
+
|
| 918 |
+
try:
|
| 919 |
+
# Use passed-in region when set, else env, else default (align with AssumeRole path)
|
| 920 |
+
region = region or "us-east-1"
|
| 921 |
+
|
| 922 |
+
# Check if we need to do cross-account role assumption
|
| 923 |
+
if aws_role_name != irsa_role_arn:
|
| 924 |
+
sts_response = self._handle_irsa_cross_account(
|
| 925 |
+
irsa_role_arn,
|
| 926 |
+
aws_role_name,
|
| 927 |
+
aws_session_name,
|
| 928 |
+
region,
|
| 929 |
+
web_identity_token_file,
|
| 930 |
+
aws_external_id,
|
| 931 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 932 |
+
ssl_verify=ssl_verify,
|
| 933 |
+
)
|
| 934 |
+
else:
|
| 935 |
+
sts_response = self._handle_irsa_same_account(
|
| 936 |
+
aws_role_name,
|
| 937 |
+
aws_session_name,
|
| 938 |
+
region,
|
| 939 |
+
aws_external_id,
|
| 940 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 941 |
+
ssl_verify=ssl_verify,
|
| 942 |
+
)
|
| 943 |
+
|
| 944 |
+
return self._extract_credentials_and_ttl(sts_response)
|
| 945 |
+
|
| 946 |
+
except Exception as e:
|
| 947 |
+
verbose_logger.debug(f"Failed to assume role via IRSA: {e}")
|
| 948 |
+
if "AccessDenied" in str(
|
| 949 |
+
e
|
| 950 |
+
) and "is not authorized to perform: sts:AssumeRole" in str(e):
|
| 951 |
+
# Provide a more helpful error message for trust policy issues
|
| 952 |
+
verbose_logger.error(
|
| 953 |
+
f"Access denied when trying to assume role {aws_role_name}. "
|
| 954 |
+
f"Please ensure the trust policy of {aws_role_name} allows "
|
| 955 |
+
f"the current role to assume it. Current identity: check logs with verbose mode."
|
| 956 |
+
)
|
| 957 |
+
# Re-raise the exception instead of falling through
|
| 958 |
+
raise
|
| 959 |
+
|
| 960 |
+
# In EKS/IRSA environments, use ambient credentials (no explicit keys needed)
|
| 961 |
+
# This allows the web identity token to work automatically
|
| 962 |
+
sts_client_kwargs: dict = {"verify": self._get_ssl_verify(ssl_verify)}
|
| 963 |
+
if region is not None:
|
| 964 |
+
sts_client_kwargs["region_name"] = region
|
| 965 |
+
if aws_sts_endpoint is not None:
|
| 966 |
+
sts_client_kwargs["endpoint_url"] = aws_sts_endpoint
|
| 967 |
+
if aws_access_key_id is None and aws_secret_access_key is None:
|
| 968 |
+
with tracer.trace("boto3.client(sts)"):
|
| 969 |
+
sts_client = boto3.client("sts", **sts_client_kwargs)
|
| 970 |
+
else:
|
| 971 |
+
with tracer.trace("boto3.client(sts)"):
|
| 972 |
+
sts_client = boto3.client(
|
| 973 |
+
"sts",
|
| 974 |
+
aws_access_key_id=aws_access_key_id,
|
| 975 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 976 |
+
aws_session_token=aws_session_token,
|
| 977 |
+
**sts_client_kwargs,
|
| 978 |
+
)
|
| 979 |
+
|
| 980 |
+
assume_role_params = {
|
| 981 |
+
"RoleArn": aws_role_name,
|
| 982 |
+
"RoleSessionName": aws_session_name,
|
| 983 |
+
}
|
| 984 |
+
|
| 985 |
+
# Add ExternalId parameter if provided
|
| 986 |
+
if aws_external_id is not None:
|
| 987 |
+
assume_role_params["ExternalId"] = aws_external_id
|
| 988 |
+
|
| 989 |
+
try:
|
| 990 |
+
sts_response = sts_client.assume_role(**assume_role_params)
|
| 991 |
+
except Exception as e:
|
| 992 |
+
error_str = str(e)
|
| 993 |
+
if "AccessDenied" in error_str:
|
| 994 |
+
# Only fall back to ambient credentials if we can positively
|
| 995 |
+
# confirm the caller is already the target role (same account,
|
| 996 |
+
# partition, and role name). This avoids silently using the
|
| 997 |
+
# wrong identity when there is a genuine trust-policy or
|
| 998 |
+
# permission misconfiguration.
|
| 999 |
+
if self._is_already_running_as_role(
|
| 1000 |
+
aws_role_name, ssl_verify=ssl_verify
|
| 1001 |
+
):
|
| 1002 |
+
verbose_logger.warning(
|
| 1003 |
+
"AssumeRole failed for %s (%s). "
|
| 1004 |
+
"Caller is already running as this role; "
|
| 1005 |
+
"falling back to ambient credentials.",
|
| 1006 |
+
aws_role_name,
|
| 1007 |
+
error_str,
|
| 1008 |
+
)
|
| 1009 |
+
return self._auth_with_env_vars()
|
| 1010 |
+
# Genuine permission error — re-raise
|
| 1011 |
+
verbose_logger.error(
|
| 1012 |
+
"AssumeRole AccessDenied for %s and caller is NOT "
|
| 1013 |
+
"the same role. Re-raising. Error: %s",
|
| 1014 |
+
aws_role_name,
|
| 1015 |
+
error_str,
|
| 1016 |
+
)
|
| 1017 |
+
raise
|
| 1018 |
+
|
| 1019 |
+
# Extract the credentials from the response and convert to Session Credentials
|
| 1020 |
+
sts_credentials = sts_response["Credentials"]
|
| 1021 |
+
credentials = Credentials(
|
| 1022 |
+
access_key=sts_credentials["AccessKeyId"],
|
| 1023 |
+
secret_key=sts_credentials["SecretAccessKey"],
|
| 1024 |
+
token=sts_credentials["SessionToken"],
|
| 1025 |
+
)
|
| 1026 |
+
|
| 1027 |
+
sts_expiry = sts_credentials["Expiration"]
|
| 1028 |
+
# Convert to timezone-aware datetime for comparison
|
| 1029 |
+
current_time = datetime.now(sts_expiry.tzinfo)
|
| 1030 |
+
sts_ttl = (sts_expiry - current_time).total_seconds() - 60
|
| 1031 |
+
return credentials, sts_ttl
|
| 1032 |
+
|
| 1033 |
+
@tracer.wrap()
|
| 1034 |
+
def _auth_with_aws_profile(
|
| 1035 |
+
self, aws_profile_name: str
|
| 1036 |
+
) -> Tuple[Credentials, Optional[int]]:
|
| 1037 |
+
"""
|
| 1038 |
+
Authenticate with AWS profile
|
| 1039 |
+
"""
|
| 1040 |
+
import boto3
|
| 1041 |
+
|
| 1042 |
+
# uses auth values from AWS profile usually stored in ~/.aws/credentials
|
| 1043 |
+
with tracer.trace("boto3.Session(profile_name=aws_profile_name)"):
|
| 1044 |
+
client = boto3.Session(profile_name=aws_profile_name)
|
| 1045 |
+
return client.get_credentials(), None
|
| 1046 |
+
|
| 1047 |
+
@tracer.wrap()
|
| 1048 |
+
def _auth_with_aws_session_token(
|
| 1049 |
+
self,
|
| 1050 |
+
aws_access_key_id: str,
|
| 1051 |
+
aws_secret_access_key: str,
|
| 1052 |
+
aws_session_token: str,
|
| 1053 |
+
) -> Tuple[Credentials, Optional[int]]:
|
| 1054 |
+
"""
|
| 1055 |
+
Authenticate with AWS Session Token
|
| 1056 |
+
"""
|
| 1057 |
+
### CHECK FOR AWS SESSION TOKEN ###
|
| 1058 |
+
from botocore.credentials import Credentials
|
| 1059 |
+
|
| 1060 |
+
credentials = Credentials(
|
| 1061 |
+
access_key=aws_access_key_id,
|
| 1062 |
+
secret_key=aws_secret_access_key,
|
| 1063 |
+
token=aws_session_token,
|
| 1064 |
+
)
|
| 1065 |
+
|
| 1066 |
+
return credentials, None
|
| 1067 |
+
|
| 1068 |
+
@tracer.wrap()
|
| 1069 |
+
def _auth_with_access_key_and_secret_key(
|
| 1070 |
+
self,
|
| 1071 |
+
aws_access_key_id: str,
|
| 1072 |
+
aws_secret_access_key: str,
|
| 1073 |
+
aws_region_name: Optional[str],
|
| 1074 |
+
) -> Tuple[Credentials, Optional[int]]:
|
| 1075 |
+
"""
|
| 1076 |
+
Authenticate with AWS Access Key and Secret Key
|
| 1077 |
+
"""
|
| 1078 |
+
import boto3
|
| 1079 |
+
|
| 1080 |
+
# Check if credentials are already in cache. These credentials have no expiry time.
|
| 1081 |
+
with tracer.trace(
|
| 1082 |
+
"boto3.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name)"
|
| 1083 |
+
):
|
| 1084 |
+
session = boto3.Session(
|
| 1085 |
+
aws_access_key_id=aws_access_key_id,
|
| 1086 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 1087 |
+
region_name=aws_region_name,
|
| 1088 |
+
)
|
| 1089 |
+
|
| 1090 |
+
credentials = session.get_credentials()
|
| 1091 |
+
return credentials, self._get_default_ttl_for_boto3_credentials()
|
| 1092 |
+
|
| 1093 |
+
@tracer.wrap()
|
| 1094 |
+
def _auth_with_env_vars(self) -> Tuple[Credentials, Optional[int]]:
|
| 1095 |
+
"""
|
| 1096 |
+
Authenticate with AWS Environment Variables
|
| 1097 |
+
"""
|
| 1098 |
+
import boto3
|
| 1099 |
+
|
| 1100 |
+
with tracer.trace("boto3.Session()"):
|
| 1101 |
+
session = boto3.Session()
|
| 1102 |
+
credentials = session.get_credentials()
|
| 1103 |
+
return credentials, None
|
| 1104 |
+
|
| 1105 |
+
@tracer.wrap()
|
| 1106 |
+
def _get_default_ttl_for_boto3_credentials(self) -> int:
|
| 1107 |
+
"""
|
| 1108 |
+
Get the default TTL for boto3 credentials
|
| 1109 |
+
|
| 1110 |
+
Returns `3600-60` which is 59 minutes
|
| 1111 |
+
"""
|
| 1112 |
+
return 3600 - 60
|
| 1113 |
+
|
| 1114 |
+
def get_runtime_endpoint(
|
| 1115 |
+
self,
|
| 1116 |
+
api_base: Optional[str],
|
| 1117 |
+
aws_bedrock_runtime_endpoint: Optional[str],
|
| 1118 |
+
aws_region_name: str,
|
| 1119 |
+
endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]] = "runtime",
|
| 1120 |
+
) -> Tuple[str, str]:
|
| 1121 |
+
env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT")
|
| 1122 |
+
if api_base is not None:
|
| 1123 |
+
endpoint_url = api_base
|
| 1124 |
+
elif aws_bedrock_runtime_endpoint is not None and isinstance(
|
| 1125 |
+
aws_bedrock_runtime_endpoint, str
|
| 1126 |
+
):
|
| 1127 |
+
endpoint_url = aws_bedrock_runtime_endpoint
|
| 1128 |
+
elif env_aws_bedrock_runtime_endpoint and isinstance(
|
| 1129 |
+
env_aws_bedrock_runtime_endpoint, str
|
| 1130 |
+
):
|
| 1131 |
+
endpoint_url = env_aws_bedrock_runtime_endpoint
|
| 1132 |
+
else:
|
| 1133 |
+
endpoint_url = self._select_default_endpoint_url(
|
| 1134 |
+
endpoint_type=endpoint_type,
|
| 1135 |
+
aws_region_name=aws_region_name,
|
| 1136 |
+
)
|
| 1137 |
+
|
| 1138 |
+
# Determine proxy_endpoint_url
|
| 1139 |
+
if aws_bedrock_runtime_endpoint is not None and isinstance(
|
| 1140 |
+
aws_bedrock_runtime_endpoint, str
|
| 1141 |
+
):
|
| 1142 |
+
proxy_endpoint_url = aws_bedrock_runtime_endpoint
|
| 1143 |
+
elif env_aws_bedrock_runtime_endpoint and isinstance(
|
| 1144 |
+
env_aws_bedrock_runtime_endpoint, str
|
| 1145 |
+
):
|
| 1146 |
+
proxy_endpoint_url = env_aws_bedrock_runtime_endpoint
|
| 1147 |
+
else:
|
| 1148 |
+
proxy_endpoint_url = endpoint_url
|
| 1149 |
+
|
| 1150 |
+
return endpoint_url, proxy_endpoint_url
|
| 1151 |
+
|
| 1152 |
+
def _select_default_endpoint_url(
|
| 1153 |
+
self,
|
| 1154 |
+
endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]],
|
| 1155 |
+
aws_region_name: str,
|
| 1156 |
+
) -> str:
|
| 1157 |
+
"""
|
| 1158 |
+
Select the default endpoint url based on the endpoint type
|
| 1159 |
+
|
| 1160 |
+
Default endpoint url is https://bedrock-runtime.{aws_region_name}.amazonaws.com
|
| 1161 |
+
"""
|
| 1162 |
+
if endpoint_type == "agent":
|
| 1163 |
+
return f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com"
|
| 1164 |
+
elif endpoint_type == "agentcore":
|
| 1165 |
+
return f"https://bedrock-agentcore.{aws_region_name}.amazonaws.com"
|
| 1166 |
+
else:
|
| 1167 |
+
return f"https://bedrock-runtime.{aws_region_name}.amazonaws.com"
|
| 1168 |
+
|
| 1169 |
+
def _get_boto_credentials_from_optional_params(
|
| 1170 |
+
self, optional_params: dict, model: Optional[str] = None
|
| 1171 |
+
) -> Boto3CredentialsInfo:
|
| 1172 |
+
"""
|
| 1173 |
+
Get boto3 credentials from optional params
|
| 1174 |
+
|
| 1175 |
+
Args:
|
| 1176 |
+
optional_params (dict): Optional parameters for the model call
|
| 1177 |
+
|
| 1178 |
+
Returns:
|
| 1179 |
+
Credentials: Boto3 credentials object
|
| 1180 |
+
"""
|
| 1181 |
+
try:
|
| 1182 |
+
from botocore.credentials import Credentials
|
| 1183 |
+
except ImportError:
|
| 1184 |
+
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
|
| 1185 |
+
## CREDENTIALS ##
|
| 1186 |
+
# pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them
|
| 1187 |
+
aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
|
| 1188 |
+
aws_access_key_id = optional_params.pop("aws_access_key_id", None)
|
| 1189 |
+
aws_session_token = optional_params.pop("aws_session_token", None)
|
| 1190 |
+
aws_region_name = self._get_aws_region_name(optional_params, model)
|
| 1191 |
+
optional_params.pop("aws_region_name", None)
|
| 1192 |
+
aws_role_name = optional_params.pop("aws_role_name", None)
|
| 1193 |
+
aws_session_name = optional_params.pop("aws_session_name", None)
|
| 1194 |
+
aws_profile_name = optional_params.pop("aws_profile_name", None)
|
| 1195 |
+
aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
|
| 1196 |
+
aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
|
| 1197 |
+
aws_bedrock_runtime_endpoint = optional_params.pop(
|
| 1198 |
+
"aws_bedrock_runtime_endpoint", None
|
| 1199 |
+
) # https://bedrock-runtime.{region_name}.amazonaws.com
|
| 1200 |
+
aws_external_id = optional_params.pop("aws_external_id", None)
|
| 1201 |
+
|
| 1202 |
+
credentials: Credentials = self.get_credentials(
|
| 1203 |
+
aws_access_key_id=aws_access_key_id,
|
| 1204 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 1205 |
+
aws_session_token=aws_session_token,
|
| 1206 |
+
aws_region_name=aws_region_name,
|
| 1207 |
+
aws_session_name=aws_session_name,
|
| 1208 |
+
aws_profile_name=aws_profile_name,
|
| 1209 |
+
aws_role_name=aws_role_name,
|
| 1210 |
+
aws_web_identity_token=aws_web_identity_token,
|
| 1211 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 1212 |
+
aws_external_id=aws_external_id,
|
| 1213 |
+
)
|
| 1214 |
+
|
| 1215 |
+
return Boto3CredentialsInfo(
|
| 1216 |
+
credentials=credentials,
|
| 1217 |
+
aws_region_name=aws_region_name,
|
| 1218 |
+
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
| 1219 |
+
)
|
| 1220 |
+
|
| 1221 |
+
@tracer.wrap()
|
| 1222 |
+
def get_request_headers(
|
| 1223 |
+
self,
|
| 1224 |
+
credentials: Credentials,
|
| 1225 |
+
aws_region_name: str,
|
| 1226 |
+
extra_headers: Optional[dict],
|
| 1227 |
+
endpoint_url: str,
|
| 1228 |
+
data: Union[str, bytes],
|
| 1229 |
+
headers: dict,
|
| 1230 |
+
api_key: Optional[str] = None,
|
| 1231 |
+
) -> AWSPreparedRequest:
|
| 1232 |
+
if api_key is not None:
|
| 1233 |
+
aws_bearer_token: Optional[str] = api_key
|
| 1234 |
+
else:
|
| 1235 |
+
aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
|
| 1236 |
+
|
| 1237 |
+
if aws_bearer_token:
|
| 1238 |
+
try:
|
| 1239 |
+
from botocore.awsrequest import AWSRequest
|
| 1240 |
+
except ImportError:
|
| 1241 |
+
raise ImportError(
|
| 1242 |
+
"Missing boto3 to call bedrock. Run 'pip install boto3'."
|
| 1243 |
+
)
|
| 1244 |
+
headers["Authorization"] = f"Bearer {aws_bearer_token}"
|
| 1245 |
+
request = AWSRequest(
|
| 1246 |
+
method="POST", url=endpoint_url, data=data, headers=headers
|
| 1247 |
+
)
|
| 1248 |
+
else:
|
| 1249 |
+
try:
|
| 1250 |
+
from botocore.auth import SigV4Auth
|
| 1251 |
+
from botocore.awsrequest import AWSRequest
|
| 1252 |
+
except ImportError:
|
| 1253 |
+
raise ImportError(
|
| 1254 |
+
"Missing boto3 to call bedrock. Run 'pip install boto3'."
|
| 1255 |
+
)
|
| 1256 |
+
|
| 1257 |
+
# Filter headers for AWS signature calculation
|
| 1258 |
+
# AWS SigV4 only includes specific headers in signature calculation
|
| 1259 |
+
aws_signature_headers = self._filter_headers_for_aws_signature(headers)
|
| 1260 |
+
sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name)
|
| 1261 |
+
request = AWSRequest(
|
| 1262 |
+
method="POST",
|
| 1263 |
+
url=endpoint_url,
|
| 1264 |
+
data=data,
|
| 1265 |
+
headers=aws_signature_headers,
|
| 1266 |
+
)
|
| 1267 |
+
sigv4.add_auth(request)
|
| 1268 |
+
|
| 1269 |
+
# Add back all original headers (including forwarded ones) after signature calculation
|
| 1270 |
+
for header_name, header_value in headers.items():
|
| 1271 |
+
if header_value is not None:
|
| 1272 |
+
request.headers[header_name] = header_value
|
| 1273 |
+
|
| 1274 |
+
if (
|
| 1275 |
+
extra_headers is not None and "Authorization" in extra_headers
|
| 1276 |
+
): # prevent sigv4 from overwriting the auth header
|
| 1277 |
+
request.headers["Authorization"] = extra_headers["Authorization"]
|
| 1278 |
+
prepped = request.prepare()
|
| 1279 |
+
|
| 1280 |
+
return prepped
|
| 1281 |
+
|
| 1282 |
+
def _filter_headers_for_aws_signature(self, headers: dict) -> dict:
|
| 1283 |
+
"""
|
| 1284 |
+
Filter headers to only include those that AWS SigV4 includes in signature calculation.
|
| 1285 |
+
This Fixes forwarded client headers from breaking the signature calculation.
|
| 1286 |
+
"""
|
| 1287 |
+
aws_signature_headers = {}
|
| 1288 |
+
aws_headers = {
|
| 1289 |
+
"host",
|
| 1290 |
+
"content-type",
|
| 1291 |
+
"date",
|
| 1292 |
+
"x-amz-date",
|
| 1293 |
+
"x-amz-security-token",
|
| 1294 |
+
"x-amz-content-sha256",
|
| 1295 |
+
"x-amz-algorithm",
|
| 1296 |
+
"x-amz-credential",
|
| 1297 |
+
"x-amz-signedheaders",
|
| 1298 |
+
"x-amz-signature",
|
| 1299 |
+
}
|
| 1300 |
+
|
| 1301 |
+
for header_name, header_value in headers.items():
|
| 1302 |
+
if header_value is None:
|
| 1303 |
+
continue
|
| 1304 |
+
header_lower = header_name.lower()
|
| 1305 |
+
if (
|
| 1306 |
+
header_lower in aws_headers
|
| 1307 |
+
or header_lower.startswith("x-amz-")
|
| 1308 |
+
or header_lower.startswith("x-amzn-")
|
| 1309 |
+
):
|
| 1310 |
+
aws_signature_headers[header_name] = header_value
|
| 1311 |
+
|
| 1312 |
+
return aws_signature_headers
|
| 1313 |
+
|
| 1314 |
+
def _sign_request(
|
| 1315 |
+
self,
|
| 1316 |
+
service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"],
|
| 1317 |
+
headers: dict,
|
| 1318 |
+
optional_params: dict,
|
| 1319 |
+
request_data: dict,
|
| 1320 |
+
api_base: str,
|
| 1321 |
+
model: Optional[str] = None,
|
| 1322 |
+
stream: Optional[bool] = None,
|
| 1323 |
+
fake_stream: Optional[bool] = None,
|
| 1324 |
+
api_key: Optional[str] = None,
|
| 1325 |
+
) -> Tuple[dict, Optional[bytes]]:
|
| 1326 |
+
"""
|
| 1327 |
+
Sign a request for Bedrock or Sagemaker
|
| 1328 |
+
|
| 1329 |
+
Returns:
|
| 1330 |
+
Tuple[dict, Optional[str]]: A tuple containing the headers and the json str body of the request
|
| 1331 |
+
"""
|
| 1332 |
+
if api_key is not None:
|
| 1333 |
+
aws_bearer_token: Optional[str] = api_key
|
| 1334 |
+
else:
|
| 1335 |
+
aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
|
| 1336 |
+
|
| 1337 |
+
# If aws bearer token is set, use it directly in the header
|
| 1338 |
+
if aws_bearer_token:
|
| 1339 |
+
headers = headers or {}
|
| 1340 |
+
headers["Content-Type"] = "application/json"
|
| 1341 |
+
headers["Authorization"] = f"Bearer {aws_bearer_token}"
|
| 1342 |
+
return headers, json.dumps(request_data).encode()
|
| 1343 |
+
|
| 1344 |
+
# If no bearer token is set, proceed with the existing SigV4 authentication
|
| 1345 |
+
try:
|
| 1346 |
+
from botocore.auth import SigV4Auth
|
| 1347 |
+
from botocore.awsrequest import AWSRequest
|
| 1348 |
+
from botocore.credentials import Credentials
|
| 1349 |
+
except ImportError:
|
| 1350 |
+
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
|
| 1351 |
+
|
| 1352 |
+
## CREDENTIALS ##
|
| 1353 |
+
# pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
|
| 1354 |
+
aws_secret_access_key = optional_params.get("aws_secret_access_key", None)
|
| 1355 |
+
aws_access_key_id = optional_params.get("aws_access_key_id", None)
|
| 1356 |
+
aws_session_token = optional_params.get("aws_session_token", None)
|
| 1357 |
+
aws_role_name = optional_params.get("aws_role_name", None)
|
| 1358 |
+
aws_session_name = optional_params.get("aws_session_name", None)
|
| 1359 |
+
aws_profile_name = optional_params.get("aws_profile_name", None)
|
| 1360 |
+
aws_web_identity_token = optional_params.get("aws_web_identity_token", None)
|
| 1361 |
+
aws_sts_endpoint = optional_params.get("aws_sts_endpoint", None)
|
| 1362 |
+
aws_external_id = optional_params.get("aws_external_id", None)
|
| 1363 |
+
aws_region_name = self._get_aws_region_name(
|
| 1364 |
+
optional_params=optional_params, model=model
|
| 1365 |
+
)
|
| 1366 |
+
|
| 1367 |
+
credentials: Credentials = self.get_credentials(
|
| 1368 |
+
aws_access_key_id=aws_access_key_id,
|
| 1369 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 1370 |
+
aws_session_token=aws_session_token,
|
| 1371 |
+
aws_region_name=aws_region_name,
|
| 1372 |
+
aws_session_name=aws_session_name,
|
| 1373 |
+
aws_profile_name=aws_profile_name,
|
| 1374 |
+
aws_role_name=aws_role_name,
|
| 1375 |
+
aws_web_identity_token=aws_web_identity_token,
|
| 1376 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 1377 |
+
aws_external_id=aws_external_id,
|
| 1378 |
+
)
|
| 1379 |
+
|
| 1380 |
+
sigv4 = SigV4Auth(credentials, service_name, aws_region_name)
|
| 1381 |
+
if headers is not None:
|
| 1382 |
+
headers = {"Content-Type": "application/json", **headers}
|
| 1383 |
+
else:
|
| 1384 |
+
headers = {"Content-Type": "application/json"}
|
| 1385 |
+
|
| 1386 |
+
aws_signature_headers = self._filter_headers_for_aws_signature(headers)
|
| 1387 |
+
request = AWSRequest(
|
| 1388 |
+
method="POST",
|
| 1389 |
+
url=api_base,
|
| 1390 |
+
data=json.dumps(request_data),
|
| 1391 |
+
headers=aws_signature_headers,
|
| 1392 |
+
)
|
| 1393 |
+
sigv4.add_auth(request)
|
| 1394 |
+
|
| 1395 |
+
request_headers_dict = dict(request.headers)
|
| 1396 |
+
# Add back original headers after signing. Only headers in SignedHeaders
|
| 1397 |
+
# are integrity-protected; forwarded headers (x-forwarded-*) must remain unsigned.
|
| 1398 |
+
for header_name, header_value in headers.items():
|
| 1399 |
+
if header_value is not None:
|
| 1400 |
+
request_headers_dict[header_name] = header_value
|
| 1401 |
+
if (
|
| 1402 |
+
headers is not None and "Authorization" in headers
|
| 1403 |
+
): # prevent sigv4 from overwriting the auth header
|
| 1404 |
+
request_headers_dict["Authorization"] = headers["Authorization"]
|
| 1405 |
+
|
| 1406 |
+
return request_headers_dict, request.body
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/handler.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from openai.types.batch import BatchRequestCounts
|
| 2 |
+
from openai.types.batch import Metadata as OpenAIBatchMetadata
|
| 3 |
+
|
| 4 |
+
from litellm.types.utils import LiteLLMBatch
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class BedrockBatchesHandler:
|
| 8 |
+
"""
|
| 9 |
+
Handler for Bedrock Batches.
|
| 10 |
+
|
| 11 |
+
Specific providers/models needed some special handling.
|
| 12 |
+
|
| 13 |
+
E.g. Twelve Labs Embedding Async Invoke
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
@staticmethod
|
| 17 |
+
def _handle_async_invoke_status(
|
| 18 |
+
batch_id: str, aws_region_name: str, logging_obj=None, **kwargs
|
| 19 |
+
) -> "LiteLLMBatch":
|
| 20 |
+
"""
|
| 21 |
+
Handle async invoke status check for AWS Bedrock.
|
| 22 |
+
|
| 23 |
+
This is for Twelve Labs Embedding Async Invoke.
|
| 24 |
+
|
| 25 |
+
Args:
|
| 26 |
+
batch_id: The async invoke ARN
|
| 27 |
+
aws_region_name: AWS region name
|
| 28 |
+
**kwargs: Additional parameters
|
| 29 |
+
|
| 30 |
+
Returns:
|
| 31 |
+
dict: Status information including status, output_file_id (S3 URL), etc.
|
| 32 |
+
"""
|
| 33 |
+
import asyncio
|
| 34 |
+
|
| 35 |
+
from litellm.llms.bedrock.embed.embedding import BedrockEmbedding
|
| 36 |
+
|
| 37 |
+
async def _async_get_status():
|
| 38 |
+
# Create embedding handler instance
|
| 39 |
+
embedding_handler = BedrockEmbedding()
|
| 40 |
+
|
| 41 |
+
# Get the status of the async invoke job
|
| 42 |
+
status_response = await embedding_handler._get_async_invoke_status(
|
| 43 |
+
invocation_arn=batch_id,
|
| 44 |
+
aws_region_name=aws_region_name,
|
| 45 |
+
logging_obj=logging_obj,
|
| 46 |
+
**kwargs,
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
# Transform response to a LiteLLMBatch object
|
| 50 |
+
from litellm.types.utils import LiteLLMBatch
|
| 51 |
+
|
| 52 |
+
openai_batch_metadata: OpenAIBatchMetadata = {
|
| 53 |
+
"output_file_id": status_response["outputDataConfig"][
|
| 54 |
+
"s3OutputDataConfig"
|
| 55 |
+
]["s3Uri"],
|
| 56 |
+
"failure_message": status_response.get("failureMessage") or "",
|
| 57 |
+
"model_arn": status_response["modelArn"],
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
result = LiteLLMBatch(
|
| 61 |
+
id=status_response["invocationArn"],
|
| 62 |
+
object="batch",
|
| 63 |
+
status=status_response["status"],
|
| 64 |
+
created_at=status_response["submitTime"],
|
| 65 |
+
in_progress_at=status_response["lastModifiedTime"],
|
| 66 |
+
completed_at=status_response.get("endTime"),
|
| 67 |
+
failed_at=status_response.get("endTime")
|
| 68 |
+
if status_response["status"] == "failed"
|
| 69 |
+
else None,
|
| 70 |
+
request_counts=BatchRequestCounts(
|
| 71 |
+
total=1,
|
| 72 |
+
completed=1 if status_response["status"] == "completed" else 0,
|
| 73 |
+
failed=1 if status_response["status"] == "failed" else 0,
|
| 74 |
+
),
|
| 75 |
+
metadata=openai_batch_metadata,
|
| 76 |
+
completion_window="24h",
|
| 77 |
+
endpoint="/v1/embeddings",
|
| 78 |
+
input_file_id="",
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
return result
|
| 82 |
+
|
| 83 |
+
# Since this function is called from within an async context via run_in_executor,
|
| 84 |
+
# we need to create a new event loop in a thread to avoid conflicts
|
| 85 |
+
import concurrent.futures
|
| 86 |
+
|
| 87 |
+
def run_in_thread():
|
| 88 |
+
new_loop = asyncio.new_event_loop()
|
| 89 |
+
asyncio.set_event_loop(new_loop)
|
| 90 |
+
try:
|
| 91 |
+
return new_loop.run_until_complete(_async_get_status())
|
| 92 |
+
finally:
|
| 93 |
+
new_loop.close()
|
| 94 |
+
|
| 95 |
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
| 96 |
+
future = executor.submit(run_in_thread)
|
| 97 |
+
return future.result()
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/transformation.py
ADDED
|
@@ -0,0 +1,549 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import time
|
| 3 |
+
from typing import Any, Dict, List, Literal, Optional, Union, cast
|
| 4 |
+
|
| 5 |
+
from httpx import Headers, Response
|
| 6 |
+
|
| 7 |
+
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
|
| 8 |
+
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
| 9 |
+
from litellm.secret_managers.main import get_secret_str
|
| 10 |
+
from litellm.types.llms.bedrock import (
|
| 11 |
+
BedrockCreateBatchRequest,
|
| 12 |
+
BedrockCreateBatchResponse,
|
| 13 |
+
BedrockInputDataConfig,
|
| 14 |
+
BedrockOutputDataConfig,
|
| 15 |
+
BedrockS3InputDataConfig,
|
| 16 |
+
BedrockS3OutputDataConfig,
|
| 17 |
+
)
|
| 18 |
+
from litellm.types.llms.openai import (
|
| 19 |
+
AllMessageValues,
|
| 20 |
+
CreateBatchRequest,
|
| 21 |
+
)
|
| 22 |
+
from litellm.types.utils import LiteLLMBatch, LlmProviders
|
| 23 |
+
|
| 24 |
+
from ..base_aws_llm import BaseAWSLLM
|
| 25 |
+
from ..common_utils import CommonBatchFilesUtils
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
|
| 29 |
+
"""
|
| 30 |
+
Config for Bedrock Batches - handles batch job creation and management for Bedrock
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
def __init__(self):
|
| 34 |
+
super().__init__()
|
| 35 |
+
self.common_utils = CommonBatchFilesUtils()
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def custom_llm_provider(self) -> LlmProviders:
|
| 39 |
+
return LlmProviders.BEDROCK
|
| 40 |
+
|
| 41 |
+
def validate_environment(
|
| 42 |
+
self,
|
| 43 |
+
headers: dict,
|
| 44 |
+
model: str,
|
| 45 |
+
messages: List[AllMessageValues],
|
| 46 |
+
optional_params: dict,
|
| 47 |
+
litellm_params: dict,
|
| 48 |
+
api_key: Optional[str] = None,
|
| 49 |
+
api_base: Optional[str] = None,
|
| 50 |
+
) -> dict:
|
| 51 |
+
"""
|
| 52 |
+
Validate and prepare environment for Bedrock batch requests.
|
| 53 |
+
AWS credentials are handled by BaseAWSLLM.
|
| 54 |
+
"""
|
| 55 |
+
# Add any Bedrock-specific headers if needed
|
| 56 |
+
return headers
|
| 57 |
+
|
| 58 |
+
def get_complete_batch_url(
|
| 59 |
+
self,
|
| 60 |
+
api_base: Optional[str],
|
| 61 |
+
api_key: Optional[str],
|
| 62 |
+
model: str,
|
| 63 |
+
optional_params: Dict,
|
| 64 |
+
litellm_params: Dict,
|
| 65 |
+
data: CreateBatchRequest,
|
| 66 |
+
) -> str:
|
| 67 |
+
"""
|
| 68 |
+
Get the complete URL for Bedrock batch creation.
|
| 69 |
+
Bedrock batch jobs are created via the model invocation job API.
|
| 70 |
+
"""
|
| 71 |
+
aws_region_name = self._get_aws_region_name(optional_params, model)
|
| 72 |
+
|
| 73 |
+
# Bedrock model invocation job endpoint
|
| 74 |
+
# Format: https://bedrock.{region}.amazonaws.com/model-invocation-job
|
| 75 |
+
bedrock_endpoint = (
|
| 76 |
+
f"https://bedrock.{aws_region_name}.amazonaws.com/model-invocation-job"
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
return bedrock_endpoint
|
| 80 |
+
|
| 81 |
+
def transform_create_batch_request(
|
| 82 |
+
self,
|
| 83 |
+
model: str,
|
| 84 |
+
create_batch_data: CreateBatchRequest,
|
| 85 |
+
optional_params: dict,
|
| 86 |
+
litellm_params: dict,
|
| 87 |
+
) -> Dict[str, Any]:
|
| 88 |
+
"""
|
| 89 |
+
Transform the batch creation request to Bedrock format.
|
| 90 |
+
|
| 91 |
+
Bedrock batch inference requires:
|
| 92 |
+
- modelId: The Bedrock model ID
|
| 93 |
+
- jobName: Unique name for the batch job
|
| 94 |
+
- inputDataConfig: Configuration for input data (S3 location)
|
| 95 |
+
- outputDataConfig: Configuration for output data (S3 location)
|
| 96 |
+
- roleArn: IAM role ARN for the batch job
|
| 97 |
+
"""
|
| 98 |
+
# Get required parameters
|
| 99 |
+
input_file_id = create_batch_data.get("input_file_id")
|
| 100 |
+
if not input_file_id:
|
| 101 |
+
raise ValueError("input_file_id is required for Bedrock batch creation")
|
| 102 |
+
|
| 103 |
+
# Extract S3 information from file ID using common utility
|
| 104 |
+
input_bucket, input_key = self.common_utils.parse_s3_uri(input_file_id)
|
| 105 |
+
|
| 106 |
+
# Get output S3 configuration
|
| 107 |
+
output_bucket = litellm_params.get("s3_output_bucket_name") or os.getenv(
|
| 108 |
+
"AWS_S3_OUTPUT_BUCKET_NAME"
|
| 109 |
+
)
|
| 110 |
+
if not output_bucket:
|
| 111 |
+
# Use same bucket as input if no output bucket specified
|
| 112 |
+
output_bucket = input_bucket
|
| 113 |
+
|
| 114 |
+
# Get IAM role ARN
|
| 115 |
+
role_arn = (
|
| 116 |
+
litellm_params.get("aws_batch_role_arn")
|
| 117 |
+
or optional_params.get("aws_batch_role_arn")
|
| 118 |
+
or os.getenv("AWS_BATCH_ROLE_ARN")
|
| 119 |
+
)
|
| 120 |
+
if not role_arn:
|
| 121 |
+
raise ValueError(
|
| 122 |
+
"AWS IAM role ARN is required for Bedrock batch jobs. "
|
| 123 |
+
"Set 'aws_batch_role_arn' in litellm_params or AWS_BATCH_ROLE_ARN env var"
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
if not model:
|
| 127 |
+
raise ValueError(
|
| 128 |
+
"Could not determine Bedrock model ID. Please pass `model` in your request body."
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
# Generate job name with the correct model ID using common utility
|
| 132 |
+
job_name = self.common_utils.generate_unique_job_name(model, prefix="litellm")
|
| 133 |
+
output_key = f"litellm-batch-outputs/{job_name}/"
|
| 134 |
+
|
| 135 |
+
# Build input data config
|
| 136 |
+
input_data_config: BedrockInputDataConfig = {
|
| 137 |
+
"s3InputDataConfig": BedrockS3InputDataConfig(
|
| 138 |
+
s3Uri=f"s3://{input_bucket}/{input_key}"
|
| 139 |
+
)
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
# Build output data config
|
| 143 |
+
s3_output_config: BedrockS3OutputDataConfig = BedrockS3OutputDataConfig(
|
| 144 |
+
s3Uri=f"s3://{output_bucket}/{output_key}"
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
# Add optional KMS encryption key ID if provided
|
| 148 |
+
s3_encryption_key_id = litellm_params.get(
|
| 149 |
+
"s3_encryption_key_id"
|
| 150 |
+
) or get_secret_str("AWS_S3_ENCRYPTION_KEY_ID")
|
| 151 |
+
if s3_encryption_key_id:
|
| 152 |
+
s3_output_config["s3EncryptionKeyId"] = s3_encryption_key_id
|
| 153 |
+
|
| 154 |
+
output_data_config: BedrockOutputDataConfig = {
|
| 155 |
+
"s3OutputDataConfig": s3_output_config
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
# Create Bedrock batch request with proper typing
|
| 159 |
+
bedrock_request: BedrockCreateBatchRequest = {
|
| 160 |
+
"modelId": model,
|
| 161 |
+
"jobName": job_name,
|
| 162 |
+
"inputDataConfig": input_data_config,
|
| 163 |
+
"outputDataConfig": output_data_config,
|
| 164 |
+
"roleArn": role_arn,
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
# Add optional parameters if provided
|
| 168 |
+
completion_window = create_batch_data.get("completion_window")
|
| 169 |
+
if completion_window:
|
| 170 |
+
# Map OpenAI completion window to Bedrock timeout
|
| 171 |
+
# OpenAI uses "24h", Bedrock expects timeout in hours
|
| 172 |
+
if completion_window == "24h":
|
| 173 |
+
bedrock_request["timeoutDurationInHours"] = 24
|
| 174 |
+
|
| 175 |
+
# For Bedrock, we need to return a pre-signed request with AWS auth headers
|
| 176 |
+
# Use common utility for AWS signing
|
| 177 |
+
endpoint_url = f"https://bedrock.{self._get_aws_region_name(optional_params, model)}.amazonaws.com/model-invocation-job"
|
| 178 |
+
signed_headers, signed_data = self.common_utils.sign_aws_request(
|
| 179 |
+
service_name="bedrock",
|
| 180 |
+
data=bedrock_request,
|
| 181 |
+
endpoint_url=endpoint_url,
|
| 182 |
+
optional_params=optional_params,
|
| 183 |
+
method="POST",
|
| 184 |
+
)
|
| 185 |
+
|
| 186 |
+
# Return a pre-signed request format that the HTTP handler can use
|
| 187 |
+
return {
|
| 188 |
+
"method": "POST",
|
| 189 |
+
"url": endpoint_url,
|
| 190 |
+
"headers": signed_headers,
|
| 191 |
+
"data": signed_data.decode("utf-8"),
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
def transform_create_batch_response(
|
| 195 |
+
self,
|
| 196 |
+
model: Optional[str],
|
| 197 |
+
raw_response: Response,
|
| 198 |
+
logging_obj: Any,
|
| 199 |
+
litellm_params: dict,
|
| 200 |
+
) -> LiteLLMBatch:
|
| 201 |
+
"""
|
| 202 |
+
Transform Bedrock batch creation response to LiteLLM format.
|
| 203 |
+
"""
|
| 204 |
+
try:
|
| 205 |
+
response_data: BedrockCreateBatchResponse = raw_response.json()
|
| 206 |
+
except Exception as e:
|
| 207 |
+
raise ValueError(f"Failed to parse Bedrock batch response: {e}")
|
| 208 |
+
|
| 209 |
+
# Extract information from typed Bedrock response
|
| 210 |
+
job_arn = response_data.get("jobArn", "")
|
| 211 |
+
status_str: str = str(response_data.get("status", "Submitted"))
|
| 212 |
+
|
| 213 |
+
# Map Bedrock status to OpenAI-compatible status
|
| 214 |
+
status_mapping: Dict[str, str] = {
|
| 215 |
+
"Submitted": "validating",
|
| 216 |
+
"Validating": "validating",
|
| 217 |
+
"Scheduled": "in_progress",
|
| 218 |
+
"InProgress": "in_progress",
|
| 219 |
+
"PartiallyCompleted": "completed",
|
| 220 |
+
"Completed": "completed",
|
| 221 |
+
"Failed": "failed",
|
| 222 |
+
"Stopping": "cancelling",
|
| 223 |
+
"Stopped": "cancelled",
|
| 224 |
+
"Expired": "expired",
|
| 225 |
+
}
|
| 226 |
+
|
| 227 |
+
openai_status = cast(
|
| 228 |
+
Literal[
|
| 229 |
+
"validating",
|
| 230 |
+
"failed",
|
| 231 |
+
"in_progress",
|
| 232 |
+
"finalizing",
|
| 233 |
+
"completed",
|
| 234 |
+
"expired",
|
| 235 |
+
"cancelling",
|
| 236 |
+
"cancelled",
|
| 237 |
+
],
|
| 238 |
+
status_mapping.get(status_str, "validating"),
|
| 239 |
+
)
|
| 240 |
+
|
| 241 |
+
# Get original request data from litellm_params if available
|
| 242 |
+
original_request = litellm_params.get("original_batch_request", {})
|
| 243 |
+
|
| 244 |
+
# Create LiteLLM batch object
|
| 245 |
+
return LiteLLMBatch(
|
| 246 |
+
id=job_arn, # Use ARN as the batch ID
|
| 247 |
+
object="batch",
|
| 248 |
+
endpoint=original_request.get("endpoint", "/v1/chat/completions"),
|
| 249 |
+
errors=None,
|
| 250 |
+
input_file_id=original_request.get("input_file_id", ""),
|
| 251 |
+
completion_window=original_request.get("completion_window", "24h"),
|
| 252 |
+
status=openai_status,
|
| 253 |
+
output_file_id=None, # Will be populated when job completes
|
| 254 |
+
error_file_id=None,
|
| 255 |
+
created_at=int(time.time()),
|
| 256 |
+
in_progress_at=int(time.time()) if status_str == "InProgress" else None,
|
| 257 |
+
expires_at=None,
|
| 258 |
+
finalizing_at=None,
|
| 259 |
+
completed_at=None,
|
| 260 |
+
failed_at=None,
|
| 261 |
+
expired_at=None,
|
| 262 |
+
cancelling_at=None,
|
| 263 |
+
cancelled_at=None,
|
| 264 |
+
request_counts=None,
|
| 265 |
+
metadata=original_request.get("metadata", {}),
|
| 266 |
+
)
|
| 267 |
+
|
| 268 |
+
def transform_retrieve_batch_request(
|
| 269 |
+
self,
|
| 270 |
+
batch_id: str,
|
| 271 |
+
optional_params: dict,
|
| 272 |
+
litellm_params: dict,
|
| 273 |
+
) -> Dict[str, Any]:
|
| 274 |
+
"""
|
| 275 |
+
Transform batch retrieval request for Bedrock.
|
| 276 |
+
|
| 277 |
+
Args:
|
| 278 |
+
batch_id: Bedrock job ARN
|
| 279 |
+
optional_params: Optional parameters
|
| 280 |
+
litellm_params: LiteLLM parameters
|
| 281 |
+
|
| 282 |
+
Returns:
|
| 283 |
+
Transformed request data for Bedrock GetModelInvocationJob API
|
| 284 |
+
"""
|
| 285 |
+
# For Bedrock, batch_id should be the full job ARN
|
| 286 |
+
# The GetModelInvocationJob API expects the full ARN as the identifier
|
| 287 |
+
if not batch_id.startswith("arn:aws:bedrock:"):
|
| 288 |
+
raise ValueError(f"Invalid batch_id format. Expected ARN, got: {batch_id}")
|
| 289 |
+
|
| 290 |
+
# Extract the job identifier from the ARN - use the full ARN path part
|
| 291 |
+
# ARN format: arn:aws:bedrock:region:account:model-invocation-job/job-name
|
| 292 |
+
arn_parts = batch_id.split(":")
|
| 293 |
+
if len(arn_parts) < 6:
|
| 294 |
+
raise ValueError(f"Invalid ARN format: {batch_id}")
|
| 295 |
+
|
| 296 |
+
region = arn_parts[3]
|
| 297 |
+
# arn_parts[5] contains "model-invocation-job/{jobId}"
|
| 298 |
+
|
| 299 |
+
# Build the endpoint URL for GetModelInvocationJob
|
| 300 |
+
# AWS API format: GET /model-invocation-job/{jobIdentifier}
|
| 301 |
+
# Use the FULL ARN as jobIdentifier and URL-encode it (includes ':' and '/')
|
| 302 |
+
import urllib.parse as _ul
|
| 303 |
+
|
| 304 |
+
encoded_arn = _ul.quote(batch_id, safe="")
|
| 305 |
+
endpoint_url = (
|
| 306 |
+
f"https://bedrock.{region}.amazonaws.com/model-invocation-job/{encoded_arn}"
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
# Use common utility for AWS signing
|
| 310 |
+
signed_headers, _ = self.common_utils.sign_aws_request(
|
| 311 |
+
service_name="bedrock",
|
| 312 |
+
data={}, # GET request has no body
|
| 313 |
+
endpoint_url=endpoint_url,
|
| 314 |
+
optional_params=optional_params,
|
| 315 |
+
method="GET",
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
# Return pre-signed request format
|
| 319 |
+
return {
|
| 320 |
+
"method": "GET",
|
| 321 |
+
"url": endpoint_url,
|
| 322 |
+
"headers": signed_headers,
|
| 323 |
+
"data": None,
|
| 324 |
+
}
|
| 325 |
+
|
| 326 |
+
def _parse_timestamps_and_status(self, response_data, status_str: str):
|
| 327 |
+
"""Helper to parse timestamps based on status."""
|
| 328 |
+
import datetime
|
| 329 |
+
|
| 330 |
+
def parse_timestamp(ts_str: Optional[str]) -> Optional[int]:
|
| 331 |
+
if not ts_str:
|
| 332 |
+
return None
|
| 333 |
+
try:
|
| 334 |
+
dt = datetime.datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
|
| 335 |
+
return int(dt.timestamp())
|
| 336 |
+
except Exception:
|
| 337 |
+
return None
|
| 338 |
+
|
| 339 |
+
created_at = parse_timestamp(
|
| 340 |
+
str(response_data.get("submitTime"))
|
| 341 |
+
if response_data.get("submitTime") is not None
|
| 342 |
+
else None
|
| 343 |
+
)
|
| 344 |
+
in_progress_states = {"InProgress", "Validating", "Scheduled"}
|
| 345 |
+
in_progress_at = (
|
| 346 |
+
parse_timestamp(
|
| 347 |
+
str(response_data.get("lastModifiedTime"))
|
| 348 |
+
if response_data.get("lastModifiedTime") is not None
|
| 349 |
+
else None
|
| 350 |
+
)
|
| 351 |
+
if status_str in in_progress_states
|
| 352 |
+
else None
|
| 353 |
+
)
|
| 354 |
+
completed_at = (
|
| 355 |
+
parse_timestamp(
|
| 356 |
+
str(response_data.get("endTime"))
|
| 357 |
+
if response_data.get("endTime") is not None
|
| 358 |
+
else None
|
| 359 |
+
)
|
| 360 |
+
if status_str in {"Completed", "PartiallyCompleted"}
|
| 361 |
+
else None
|
| 362 |
+
)
|
| 363 |
+
failed_at = (
|
| 364 |
+
parse_timestamp(
|
| 365 |
+
str(response_data.get("endTime"))
|
| 366 |
+
if response_data.get("endTime") is not None
|
| 367 |
+
else None
|
| 368 |
+
)
|
| 369 |
+
if status_str == "Failed"
|
| 370 |
+
else None
|
| 371 |
+
)
|
| 372 |
+
cancelled_at = (
|
| 373 |
+
parse_timestamp(
|
| 374 |
+
str(response_data.get("endTime"))
|
| 375 |
+
if response_data.get("endTime") is not None
|
| 376 |
+
else None
|
| 377 |
+
)
|
| 378 |
+
if status_str == "Stopped"
|
| 379 |
+
else None
|
| 380 |
+
)
|
| 381 |
+
expires_at = parse_timestamp(
|
| 382 |
+
str(response_data.get("jobExpirationTime"))
|
| 383 |
+
if response_data.get("jobExpirationTime") is not None
|
| 384 |
+
else None
|
| 385 |
+
)
|
| 386 |
+
|
| 387 |
+
return (
|
| 388 |
+
created_at,
|
| 389 |
+
in_progress_at,
|
| 390 |
+
completed_at,
|
| 391 |
+
failed_at,
|
| 392 |
+
cancelled_at,
|
| 393 |
+
expires_at,
|
| 394 |
+
)
|
| 395 |
+
|
| 396 |
+
def _extract_file_configs(self, response_data):
|
| 397 |
+
"""Helper to extract input and output file configurations."""
|
| 398 |
+
# Extract input file ID
|
| 399 |
+
input_file_id = ""
|
| 400 |
+
input_data_config = response_data.get("inputDataConfig", {})
|
| 401 |
+
if isinstance(input_data_config, dict):
|
| 402 |
+
s3_input_config = input_data_config.get("s3InputDataConfig", {})
|
| 403 |
+
if isinstance(s3_input_config, dict):
|
| 404 |
+
input_file_id = s3_input_config.get("s3Uri", "")
|
| 405 |
+
|
| 406 |
+
# Extract output file ID
|
| 407 |
+
output_file_id = None
|
| 408 |
+
output_data_config = response_data.get("outputDataConfig", {})
|
| 409 |
+
if isinstance(output_data_config, dict):
|
| 410 |
+
s3_output_config = output_data_config.get("s3OutputDataConfig", {})
|
| 411 |
+
if isinstance(s3_output_config, dict):
|
| 412 |
+
output_file_id = s3_output_config.get("s3Uri", "")
|
| 413 |
+
|
| 414 |
+
return input_file_id, output_file_id
|
| 415 |
+
|
| 416 |
+
def _extract_errors_and_metadata(self, response_data, raw_response):
|
| 417 |
+
"""Helper to extract errors and enriched metadata."""
|
| 418 |
+
# Extract errors
|
| 419 |
+
message = response_data.get("message")
|
| 420 |
+
errors = None
|
| 421 |
+
if message:
|
| 422 |
+
from openai.types.batch import Errors
|
| 423 |
+
from openai.types.batch_error import BatchError
|
| 424 |
+
|
| 425 |
+
errors = Errors(
|
| 426 |
+
data=[BatchError(message=message, code=str(raw_response.status_code))],
|
| 427 |
+
object="list",
|
| 428 |
+
)
|
| 429 |
+
|
| 430 |
+
# Enrich metadata with useful Bedrock fields
|
| 431 |
+
enriched_metadata_raw: Dict[str, Any] = {
|
| 432 |
+
"jobName": response_data.get("jobName"),
|
| 433 |
+
"clientRequestToken": response_data.get("clientRequestToken"),
|
| 434 |
+
"modelId": response_data.get("modelId"),
|
| 435 |
+
"roleArn": response_data.get("roleArn"),
|
| 436 |
+
"timeoutDurationInHours": response_data.get("timeoutDurationInHours"),
|
| 437 |
+
"vpcConfig": response_data.get("vpcConfig"),
|
| 438 |
+
}
|
| 439 |
+
import json as _json
|
| 440 |
+
|
| 441 |
+
enriched_metadata: Dict[str, str] = {}
|
| 442 |
+
for _k, _v in enriched_metadata_raw.items():
|
| 443 |
+
if _v is None:
|
| 444 |
+
continue
|
| 445 |
+
if isinstance(_v, (dict, list)):
|
| 446 |
+
try:
|
| 447 |
+
enriched_metadata[_k] = _json.dumps(_v)
|
| 448 |
+
except Exception:
|
| 449 |
+
enriched_metadata[_k] = str(_v)
|
| 450 |
+
else:
|
| 451 |
+
enriched_metadata[_k] = str(_v)
|
| 452 |
+
|
| 453 |
+
return errors, enriched_metadata
|
| 454 |
+
|
| 455 |
+
def transform_retrieve_batch_response(
|
| 456 |
+
self,
|
| 457 |
+
model: Optional[str],
|
| 458 |
+
raw_response: Response,
|
| 459 |
+
logging_obj: Any,
|
| 460 |
+
litellm_params: dict,
|
| 461 |
+
) -> LiteLLMBatch:
|
| 462 |
+
"""
|
| 463 |
+
Transform Bedrock batch retrieval response to LiteLLM format.
|
| 464 |
+
"""
|
| 465 |
+
from litellm.types.llms.bedrock import BedrockGetBatchResponse
|
| 466 |
+
|
| 467 |
+
try:
|
| 468 |
+
response_data: BedrockGetBatchResponse = raw_response.json()
|
| 469 |
+
except Exception as e:
|
| 470 |
+
raise ValueError(f"Failed to parse Bedrock batch response: {e}")
|
| 471 |
+
|
| 472 |
+
job_arn = response_data.get("jobArn", "")
|
| 473 |
+
status_str: str = str(response_data.get("status", "Submitted"))
|
| 474 |
+
|
| 475 |
+
# Map Bedrock status to OpenAI-compatible status
|
| 476 |
+
status_mapping: Dict[str, str] = {
|
| 477 |
+
"Submitted": "validating",
|
| 478 |
+
"Validating": "validating",
|
| 479 |
+
"Scheduled": "in_progress",
|
| 480 |
+
"InProgress": "in_progress",
|
| 481 |
+
"PartiallyCompleted": "completed",
|
| 482 |
+
"Completed": "completed",
|
| 483 |
+
"Failed": "failed",
|
| 484 |
+
"Stopping": "cancelling",
|
| 485 |
+
"Stopped": "cancelled",
|
| 486 |
+
"Expired": "expired",
|
| 487 |
+
}
|
| 488 |
+
openai_status = cast(
|
| 489 |
+
Literal[
|
| 490 |
+
"validating",
|
| 491 |
+
"failed",
|
| 492 |
+
"in_progress",
|
| 493 |
+
"finalizing",
|
| 494 |
+
"completed",
|
| 495 |
+
"expired",
|
| 496 |
+
"cancelling",
|
| 497 |
+
"cancelled",
|
| 498 |
+
],
|
| 499 |
+
status_mapping.get(status_str, "validating"),
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
# Parse timestamps
|
| 503 |
+
(
|
| 504 |
+
created_at,
|
| 505 |
+
in_progress_at,
|
| 506 |
+
completed_at,
|
| 507 |
+
failed_at,
|
| 508 |
+
cancelled_at,
|
| 509 |
+
expires_at,
|
| 510 |
+
) = self._parse_timestamps_and_status(response_data, status_str)
|
| 511 |
+
|
| 512 |
+
# Extract file configurations
|
| 513 |
+
input_file_id, output_file_id = self._extract_file_configs(response_data)
|
| 514 |
+
|
| 515 |
+
# Extract errors and metadata
|
| 516 |
+
errors, enriched_metadata = self._extract_errors_and_metadata(
|
| 517 |
+
response_data, raw_response
|
| 518 |
+
)
|
| 519 |
+
|
| 520 |
+
return LiteLLMBatch(
|
| 521 |
+
id=job_arn,
|
| 522 |
+
object="batch",
|
| 523 |
+
endpoint="/v1/chat/completions",
|
| 524 |
+
errors=errors,
|
| 525 |
+
input_file_id=input_file_id,
|
| 526 |
+
completion_window="24h",
|
| 527 |
+
status=openai_status,
|
| 528 |
+
output_file_id=output_file_id,
|
| 529 |
+
error_file_id=None,
|
| 530 |
+
created_at=created_at or int(time.time()),
|
| 531 |
+
in_progress_at=in_progress_at,
|
| 532 |
+
expires_at=expires_at,
|
| 533 |
+
finalizing_at=None,
|
| 534 |
+
completed_at=completed_at,
|
| 535 |
+
failed_at=failed_at,
|
| 536 |
+
expired_at=None,
|
| 537 |
+
cancelling_at=None,
|
| 538 |
+
cancelled_at=cancelled_at,
|
| 539 |
+
request_counts=None,
|
| 540 |
+
metadata=enriched_metadata,
|
| 541 |
+
)
|
| 542 |
+
|
| 543 |
+
def get_error_class(
|
| 544 |
+
self, error_message: str, status_code: int, headers: Union[Dict, Headers]
|
| 545 |
+
) -> BaseLLMException:
|
| 546 |
+
"""
|
| 547 |
+
Get Bedrock-specific error class using common utility.
|
| 548 |
+
"""
|
| 549 |
+
return self.common_utils.get_error_class(error_message, status_code, headers)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
from .converse_handler import BedrockConverseLLM
|
| 4 |
+
from .invoke_handler import (
|
| 5 |
+
AmazonAnthropicClaudeStreamDecoder,
|
| 6 |
+
AmazonDeepSeekR1StreamDecoder,
|
| 7 |
+
AWSEventStreamDecoder,
|
| 8 |
+
BedrockLLM,
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def get_bedrock_event_stream_decoder(
|
| 13 |
+
invoke_provider: Optional[str], model: str, sync_stream: bool, json_mode: bool
|
| 14 |
+
):
|
| 15 |
+
if invoke_provider and invoke_provider == "anthropic":
|
| 16 |
+
decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder(
|
| 17 |
+
model=model,
|
| 18 |
+
sync_stream=sync_stream,
|
| 19 |
+
json_mode=json_mode,
|
| 20 |
+
)
|
| 21 |
+
return decoder
|
| 22 |
+
elif invoke_provider and invoke_provider == "deepseek_r1":
|
| 23 |
+
decoder = AmazonDeepSeekR1StreamDecoder(
|
| 24 |
+
model=model,
|
| 25 |
+
sync_stream=sync_stream,
|
| 26 |
+
)
|
| 27 |
+
return decoder
|
| 28 |
+
else:
|
| 29 |
+
decoder = AWSEventStreamDecoder(model=model)
|
| 30 |
+
return decoder
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .transformation import AmazonAgentCoreConfig
|
| 2 |
+
|
| 3 |
+
__all__ = ["AmazonAgentCoreConfig"]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/transformation.py
ADDED
|
@@ -0,0 +1,1037 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation for Bedrock AgentCore
|
| 3 |
+
|
| 4 |
+
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import json
|
| 8 |
+
from collections.abc import AsyncGenerator
|
| 9 |
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
|
| 10 |
+
from urllib.parse import quote
|
| 11 |
+
|
| 12 |
+
import httpx
|
| 13 |
+
|
| 14 |
+
from litellm._logging import verbose_logger
|
| 15 |
+
from litellm._uuid import uuid
|
| 16 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 17 |
+
convert_content_list_to_str,
|
| 18 |
+
)
|
| 19 |
+
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
| 20 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
| 21 |
+
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
| 22 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 23 |
+
from litellm.types.llms.bedrock_agentcore import (
|
| 24 |
+
AgentCoreMessage,
|
| 25 |
+
AgentCoreParsedResponse,
|
| 26 |
+
AgentCoreUsage,
|
| 27 |
+
)
|
| 28 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 29 |
+
from litellm.types.utils import (
|
| 30 |
+
Choices,
|
| 31 |
+
Delta,
|
| 32 |
+
Message,
|
| 33 |
+
ModelResponse,
|
| 34 |
+
ModelResponseStream,
|
| 35 |
+
StreamingChoices,
|
| 36 |
+
Usage,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
if TYPE_CHECKING:
|
| 40 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 41 |
+
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
| 42 |
+
|
| 43 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 44 |
+
else:
|
| 45 |
+
LiteLLMLoggingObj = Any
|
| 46 |
+
HTTPHandler = Any
|
| 47 |
+
AsyncHTTPHandler = Any
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
| 51 |
+
def __init__(self, **kwargs):
|
| 52 |
+
BaseConfig.__init__(self, **kwargs)
|
| 53 |
+
BaseAWSLLM.__init__(self, **kwargs)
|
| 54 |
+
|
| 55 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 56 |
+
"""
|
| 57 |
+
Bedrock AgentCore has 0 OpenAI compatible params
|
| 58 |
+
"""
|
| 59 |
+
return []
|
| 60 |
+
|
| 61 |
+
def map_openai_params(
|
| 62 |
+
self,
|
| 63 |
+
non_default_params: dict,
|
| 64 |
+
optional_params: dict,
|
| 65 |
+
model: str,
|
| 66 |
+
drop_params: bool,
|
| 67 |
+
) -> dict:
|
| 68 |
+
"""
|
| 69 |
+
Map OpenAI params to AgentCore params
|
| 70 |
+
"""
|
| 71 |
+
return optional_params
|
| 72 |
+
|
| 73 |
+
def get_complete_url(
|
| 74 |
+
self,
|
| 75 |
+
api_base: Optional[str],
|
| 76 |
+
api_key: Optional[str],
|
| 77 |
+
model: str,
|
| 78 |
+
optional_params: dict,
|
| 79 |
+
litellm_params: dict,
|
| 80 |
+
stream: Optional[bool] = None,
|
| 81 |
+
) -> str:
|
| 82 |
+
"""
|
| 83 |
+
Get the complete url for the request
|
| 84 |
+
"""
|
| 85 |
+
### SET RUNTIME ENDPOINT ###
|
| 86 |
+
aws_bedrock_runtime_endpoint = optional_params.get(
|
| 87 |
+
"aws_bedrock_runtime_endpoint", None
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
# Extract ARN from model string
|
| 91 |
+
agent_runtime_arn = self._get_agent_runtime_arn(model)
|
| 92 |
+
|
| 93 |
+
# Parse ARN to get region
|
| 94 |
+
region = self._extract_region_from_arn(agent_runtime_arn)
|
| 95 |
+
|
| 96 |
+
# Build the base endpoint URL for AgentCore
|
| 97 |
+
# Note: We don't use get_runtime_endpoint as AgentCore has its own endpoint structure
|
| 98 |
+
if aws_bedrock_runtime_endpoint:
|
| 99 |
+
base_url = aws_bedrock_runtime_endpoint
|
| 100 |
+
else:
|
| 101 |
+
base_url = f"https://bedrock-agentcore.{region}.amazonaws.com"
|
| 102 |
+
|
| 103 |
+
# Based on boto3 client.invoke_agent_runtime, the path is:
|
| 104 |
+
# /runtimes/{URL-ENCODED-ARN}/invocations?qualifier=<value>
|
| 105 |
+
encoded_arn = quote(agent_runtime_arn, safe="")
|
| 106 |
+
endpoint_url = f"{base_url}/runtimes/{encoded_arn}/invocations"
|
| 107 |
+
|
| 108 |
+
# Add qualifier as query parameter if provided
|
| 109 |
+
if "qualifier" in optional_params:
|
| 110 |
+
endpoint_url = f"{endpoint_url}?qualifier={optional_params['qualifier']}"
|
| 111 |
+
|
| 112 |
+
return endpoint_url
|
| 113 |
+
|
| 114 |
+
def sign_request(
|
| 115 |
+
self,
|
| 116 |
+
headers: dict,
|
| 117 |
+
optional_params: dict,
|
| 118 |
+
request_data: dict,
|
| 119 |
+
api_base: str,
|
| 120 |
+
api_key: Optional[str] = None,
|
| 121 |
+
model: Optional[str] = None,
|
| 122 |
+
stream: Optional[bool] = None,
|
| 123 |
+
fake_stream: Optional[bool] = None,
|
| 124 |
+
) -> Tuple[dict, Optional[bytes]]:
|
| 125 |
+
# Set Accept header required by MCP servers on AgentCore
|
| 126 |
+
# Per MCP spec (Streamable HTTP transport): client MUST include Accept header
|
| 127 |
+
# listing both application/json and text/event-stream as supported content types
|
| 128 |
+
headers["Accept"] = "application/json, text/event-stream"
|
| 129 |
+
|
| 130 |
+
# Check if api_key (bearer token) is provided for Cognito authentication
|
| 131 |
+
# Priority: api_key parameter first, then optional_params
|
| 132 |
+
jwt_token = api_key or optional_params.get("api_key")
|
| 133 |
+
if jwt_token:
|
| 134 |
+
verbose_logger.debug(
|
| 135 |
+
f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..."
|
| 136 |
+
)
|
| 137 |
+
headers["Content-Type"] = "application/json"
|
| 138 |
+
headers["Authorization"] = f"Bearer {jwt_token}"
|
| 139 |
+
# Return headers with bearer token and JSON-encoded body (not SigV4 signed)
|
| 140 |
+
return headers, json.dumps(request_data).encode()
|
| 141 |
+
|
| 142 |
+
# Otherwise, use AWS SigV4 authentication
|
| 143 |
+
verbose_logger.debug("AgentCore: Using AWS SigV4 authentication (IAM)")
|
| 144 |
+
return self._sign_request(
|
| 145 |
+
service_name="bedrock-agentcore",
|
| 146 |
+
headers=headers,
|
| 147 |
+
optional_params=optional_params,
|
| 148 |
+
request_data=request_data,
|
| 149 |
+
api_base=api_base,
|
| 150 |
+
model=model,
|
| 151 |
+
stream=stream,
|
| 152 |
+
fake_stream=fake_stream,
|
| 153 |
+
api_key=api_key,
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
def _get_agent_runtime_arn(self, model: str) -> str:
|
| 157 |
+
"""
|
| 158 |
+
Extract ARN from model string
|
| 159 |
+
model = "agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC"
|
| 160 |
+
returns: "arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC"
|
| 161 |
+
"""
|
| 162 |
+
parts = model.split("/", 1)
|
| 163 |
+
if len(parts) != 2 or parts[0] != "agentcore":
|
| 164 |
+
raise ValueError(
|
| 165 |
+
"Invalid model format. Expected format: 'model=bedrock/agentcore/arn:aws:bedrock-agentcore:region:account:runtime/runtime_id'"
|
| 166 |
+
)
|
| 167 |
+
return parts[1]
|
| 168 |
+
|
| 169 |
+
def _extract_region_from_arn(self, arn: str) -> str:
|
| 170 |
+
"""
|
| 171 |
+
Extract region from ARN
|
| 172 |
+
arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC
|
| 173 |
+
returns: us-west-2
|
| 174 |
+
"""
|
| 175 |
+
parts = arn.split(":")
|
| 176 |
+
if len(parts) >= 4:
|
| 177 |
+
return parts[3]
|
| 178 |
+
raise ValueError(f"Invalid ARN format: {arn}")
|
| 179 |
+
|
| 180 |
+
def _get_runtime_session_id(self, optional_params: dict) -> str:
|
| 181 |
+
"""
|
| 182 |
+
Get or generate runtime session ID (must be 33+ chars)
|
| 183 |
+
"""
|
| 184 |
+
session_id = optional_params.get("runtimeSessionId", None)
|
| 185 |
+
if session_id:
|
| 186 |
+
verbose_logger.debug(f"Using provided runtimeSessionId: {session_id}")
|
| 187 |
+
return session_id
|
| 188 |
+
|
| 189 |
+
# Generate a session ID with 33+ characters
|
| 190 |
+
generated_id = f"litellm-session-{str(uuid.uuid4())}"
|
| 191 |
+
verbose_logger.debug(f"Generated new session ID: {generated_id}")
|
| 192 |
+
return generated_id
|
| 193 |
+
|
| 194 |
+
def _get_runtime_user_id(self, optional_params: dict) -> Optional[str]:
|
| 195 |
+
"""
|
| 196 |
+
Get runtime user ID if provided
|
| 197 |
+
"""
|
| 198 |
+
user_id = optional_params.get("runtimeUserId", None)
|
| 199 |
+
if user_id:
|
| 200 |
+
verbose_logger.debug(f"Using provided runtimeUserId: {user_id}")
|
| 201 |
+
return user_id
|
| 202 |
+
|
| 203 |
+
def transform_request(
|
| 204 |
+
self,
|
| 205 |
+
model: str,
|
| 206 |
+
messages: List[AllMessageValues],
|
| 207 |
+
optional_params: dict,
|
| 208 |
+
litellm_params: dict,
|
| 209 |
+
headers: dict,
|
| 210 |
+
) -> dict:
|
| 211 |
+
"""
|
| 212 |
+
Transform the request to AgentCore format.
|
| 213 |
+
|
| 214 |
+
Based on boto3's implementation:
|
| 215 |
+
- Session ID goes in header: X-Amzn-Bedrock-AgentCore-Runtime-Session-Id
|
| 216 |
+
- User ID goes in header: X-Amzn-Bedrock-AgentCore-Runtime-User-Id
|
| 217 |
+
- Qualifier goes as query parameter
|
| 218 |
+
- Only the payload goes in the request body
|
| 219 |
+
|
| 220 |
+
Returns:
|
| 221 |
+
dict: Payload dict containing the prompt
|
| 222 |
+
"""
|
| 223 |
+
verbose_logger.debug(
|
| 224 |
+
f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}"
|
| 225 |
+
)
|
| 226 |
+
|
| 227 |
+
# Use the last message content as the prompt
|
| 228 |
+
prompt = convert_content_list_to_str(messages[-1])
|
| 229 |
+
|
| 230 |
+
# Create the payload - this is what goes in the body (raw JSON)
|
| 231 |
+
payload: dict = {"prompt": prompt}
|
| 232 |
+
|
| 233 |
+
# Get or generate session ID - this goes in the header
|
| 234 |
+
runtime_session_id = self._get_runtime_session_id(optional_params)
|
| 235 |
+
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = runtime_session_id
|
| 236 |
+
|
| 237 |
+
# Get user ID if provided - this goes in the header
|
| 238 |
+
runtime_user_id = self._get_runtime_user_id(optional_params)
|
| 239 |
+
if runtime_user_id:
|
| 240 |
+
headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id
|
| 241 |
+
|
| 242 |
+
# The request data is the payload dict (will be JSON encoded by the HTTP handler)
|
| 243 |
+
# Qualifier will be handled as a query parameter in get_complete_url
|
| 244 |
+
|
| 245 |
+
verbose_logger.debug(f"PAYLOAD: {payload}")
|
| 246 |
+
return payload
|
| 247 |
+
|
| 248 |
+
def _extract_sse_json(self, line: str) -> Optional[Dict]:
|
| 249 |
+
"""Extract and parse JSON from an SSE data line."""
|
| 250 |
+
if not line.startswith("data:"):
|
| 251 |
+
return None
|
| 252 |
+
|
| 253 |
+
json_str = line[5:].strip()
|
| 254 |
+
if not json_str:
|
| 255 |
+
return None
|
| 256 |
+
|
| 257 |
+
try:
|
| 258 |
+
data = json.loads(json_str)
|
| 259 |
+
# Skip non-dict data (some lines contain JSON strings)
|
| 260 |
+
return data if isinstance(data, dict) else None
|
| 261 |
+
except json.JSONDecodeError:
|
| 262 |
+
verbose_logger.debug(f"Skipping non-JSON line: {line[:100]}")
|
| 263 |
+
return None
|
| 264 |
+
|
| 265 |
+
def _extract_usage_from_event(self, event_data: Dict) -> Optional[AgentCoreUsage]:
|
| 266 |
+
"""Extract usage information from event metadata."""
|
| 267 |
+
event_payload = event_data.get("event")
|
| 268 |
+
if not event_payload:
|
| 269 |
+
return None
|
| 270 |
+
|
| 271 |
+
metadata = event_payload.get("metadata")
|
| 272 |
+
if metadata and "usage" in metadata:
|
| 273 |
+
return metadata["usage"] # type: ignore
|
| 274 |
+
|
| 275 |
+
return None
|
| 276 |
+
|
| 277 |
+
def _extract_content_delta(self, event_data: Dict) -> Optional[str]:
|
| 278 |
+
"""Extract text content from contentBlockDelta event."""
|
| 279 |
+
event_payload = event_data.get("event")
|
| 280 |
+
if not event_payload:
|
| 281 |
+
return None
|
| 282 |
+
|
| 283 |
+
content_block_delta = event_payload.get("contentBlockDelta")
|
| 284 |
+
if not content_block_delta:
|
| 285 |
+
return None
|
| 286 |
+
|
| 287 |
+
delta = content_block_delta.get("delta", {})
|
| 288 |
+
return delta.get("text")
|
| 289 |
+
|
| 290 |
+
def _extract_content_from_message(self, message: AgentCoreMessage) -> str:
|
| 291 |
+
"""
|
| 292 |
+
Extract text content from message content blocks.
|
| 293 |
+
This works for both SSE messages and JSON responses.
|
| 294 |
+
"""
|
| 295 |
+
content_list = message.get("content", [])
|
| 296 |
+
if not isinstance(content_list, list):
|
| 297 |
+
return ""
|
| 298 |
+
|
| 299 |
+
return "".join(
|
| 300 |
+
block["text"]
|
| 301 |
+
for block in content_list
|
| 302 |
+
if isinstance(block, dict) and "text" in block
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
def _calculate_usage(
|
| 306 |
+
self, model: str, messages: List[AllMessageValues], content: str
|
| 307 |
+
) -> Optional[Usage]:
|
| 308 |
+
"""
|
| 309 |
+
Calculate token usage using LiteLLM's token counter.
|
| 310 |
+
|
| 311 |
+
Args:
|
| 312 |
+
model: The model name
|
| 313 |
+
messages: Input messages
|
| 314 |
+
content: Response content
|
| 315 |
+
|
| 316 |
+
Returns:
|
| 317 |
+
Usage object with calculated tokens, or None if calculation fails
|
| 318 |
+
"""
|
| 319 |
+
try:
|
| 320 |
+
from litellm.utils import token_counter
|
| 321 |
+
|
| 322 |
+
prompt_tokens = token_counter(model=model, messages=messages)
|
| 323 |
+
completion_tokens = token_counter(
|
| 324 |
+
model=model, text=content, count_response_tokens=True
|
| 325 |
+
)
|
| 326 |
+
total_tokens = prompt_tokens + completion_tokens
|
| 327 |
+
|
| 328 |
+
verbose_logger.debug(
|
| 329 |
+
f"Calculated usage - prompt: {prompt_tokens}, completion: {completion_tokens}, total: {total_tokens}"
|
| 330 |
+
)
|
| 331 |
+
|
| 332 |
+
return Usage(
|
| 333 |
+
prompt_tokens=prompt_tokens,
|
| 334 |
+
completion_tokens=completion_tokens,
|
| 335 |
+
total_tokens=total_tokens,
|
| 336 |
+
)
|
| 337 |
+
except Exception as e:
|
| 338 |
+
verbose_logger.warning(f"Failed to calculate token usage: {str(e)}")
|
| 339 |
+
return None
|
| 340 |
+
|
| 341 |
+
def _parse_json_response(self, response_json: dict) -> AgentCoreParsedResponse:
|
| 342 |
+
"""
|
| 343 |
+
Parse direct JSON response (non-streaming).
|
| 344 |
+
|
| 345 |
+
Supports multiple agent response schemas:
|
| 346 |
+
1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore
|
| 347 |
+
2. {"response": [{"text": "..."}]} - Strands agent format
|
| 348 |
+
3. {"result": "plain text"} or {"response": "plain text"} - simple string
|
| 349 |
+
4. Fallback: raw JSON as content string
|
| 350 |
+
"""
|
| 351 |
+
# Guard: if json.loads() returned a non-dict (e.g. array or primitive),
|
| 352 |
+
# skip strategy matching and fall back to raw JSON string
|
| 353 |
+
if not isinstance(response_json, dict):
|
| 354 |
+
verbose_logger.warning(
|
| 355 |
+
"AgentCore: JSON response is not a dict. "
|
| 356 |
+
"Returning raw JSON as content."
|
| 357 |
+
)
|
| 358 |
+
return AgentCoreParsedResponse(
|
| 359 |
+
content=json.dumps(response_json),
|
| 360 |
+
usage=None,
|
| 361 |
+
final_message=None,
|
| 362 |
+
)
|
| 363 |
+
|
| 364 |
+
# Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format
|
| 365 |
+
if "result" in response_json and isinstance(response_json["result"], dict):
|
| 366 |
+
result = response_json["result"]
|
| 367 |
+
content = self._extract_content_from_message(result) # type: ignore
|
| 368 |
+
return AgentCoreParsedResponse(
|
| 369 |
+
content=content,
|
| 370 |
+
usage=None,
|
| 371 |
+
final_message=result, # type: ignore
|
| 372 |
+
)
|
| 373 |
+
|
| 374 |
+
# Strategy 2: {"response": [{"text": "..."}]} - Strands agent content blocks
|
| 375 |
+
if "response" in response_json and isinstance(response_json["response"], list):
|
| 376 |
+
content = self._extract_content_from_message(
|
| 377 |
+
{"content": response_json["response"]} # type: ignore
|
| 378 |
+
)
|
| 379 |
+
return AgentCoreParsedResponse(
|
| 380 |
+
content=content,
|
| 381 |
+
usage=None,
|
| 382 |
+
final_message=None,
|
| 383 |
+
)
|
| 384 |
+
|
| 385 |
+
# Strategy 3: string values - {"result": "text"} or {"response": "text"}
|
| 386 |
+
for key in ("result", "response"):
|
| 387 |
+
val = response_json.get(key)
|
| 388 |
+
if isinstance(val, str):
|
| 389 |
+
return AgentCoreParsedResponse(
|
| 390 |
+
content=val,
|
| 391 |
+
usage=None,
|
| 392 |
+
final_message=None,
|
| 393 |
+
)
|
| 394 |
+
|
| 395 |
+
# Strategy 4: fallback - return raw JSON as content
|
| 396 |
+
verbose_logger.warning(
|
| 397 |
+
f"AgentCore: Could not extract content from JSON response keys "
|
| 398 |
+
f"{list(response_json.keys())}. Returning raw JSON as content."
|
| 399 |
+
)
|
| 400 |
+
return AgentCoreParsedResponse(
|
| 401 |
+
content=json.dumps(response_json),
|
| 402 |
+
usage=None,
|
| 403 |
+
final_message=None,
|
| 404 |
+
)
|
| 405 |
+
|
| 406 |
+
def _get_parsed_response(
|
| 407 |
+
self, raw_response: httpx.Response
|
| 408 |
+
) -> AgentCoreParsedResponse:
|
| 409 |
+
"""
|
| 410 |
+
Parse AgentCore response based on content type.
|
| 411 |
+
|
| 412 |
+
Args:
|
| 413 |
+
raw_response: Raw HTTP response from AgentCore
|
| 414 |
+
|
| 415 |
+
Returns:
|
| 416 |
+
AgentCoreParsedResponse: Parsed response data
|
| 417 |
+
"""
|
| 418 |
+
content_type = raw_response.headers.get("content-type", "").lower()
|
| 419 |
+
verbose_logger.debug(f"AgentCore response Content-Type: {content_type}")
|
| 420 |
+
|
| 421 |
+
# Parse response based on content type
|
| 422 |
+
if "application/json" in content_type:
|
| 423 |
+
# Direct JSON response
|
| 424 |
+
verbose_logger.debug("Parsing JSON response")
|
| 425 |
+
response_json = raw_response.json()
|
| 426 |
+
verbose_logger.debug(f"Response JSON: {response_json}")
|
| 427 |
+
return self._parse_json_response(response_json)
|
| 428 |
+
else:
|
| 429 |
+
# SSE stream response (text/event-stream or default)
|
| 430 |
+
verbose_logger.debug("Parsing SSE stream response")
|
| 431 |
+
response_text = raw_response.text
|
| 432 |
+
verbose_logger.debug(
|
| 433 |
+
f"AgentCore response (first 500 chars): {response_text[:500]}"
|
| 434 |
+
)
|
| 435 |
+
return self._parse_sse_stream(response_text)
|
| 436 |
+
|
| 437 |
+
def _parse_sse_stream(self, response_text: str) -> AgentCoreParsedResponse:
|
| 438 |
+
"""
|
| 439 |
+
Parse Server-Sent Events (SSE) stream format.
|
| 440 |
+
Each line starts with 'data:' followed by JSON.
|
| 441 |
+
|
| 442 |
+
Returns:
|
| 443 |
+
AgentCoreParsedResponse: Parsed response with content, usage, and message
|
| 444 |
+
"""
|
| 445 |
+
final_message: Optional[AgentCoreMessage] = None
|
| 446 |
+
usage_data: Optional[AgentCoreUsage] = None
|
| 447 |
+
content_blocks: List[str] = []
|
| 448 |
+
|
| 449 |
+
for line in response_text.strip().split("\n"):
|
| 450 |
+
line = line.strip()
|
| 451 |
+
if not line:
|
| 452 |
+
continue
|
| 453 |
+
|
| 454 |
+
data = self._extract_sse_json(line)
|
| 455 |
+
if not data:
|
| 456 |
+
continue
|
| 457 |
+
|
| 458 |
+
verbose_logger.debug(f"SSE event keys: {list(data.keys())}")
|
| 459 |
+
|
| 460 |
+
# Check for final complete message
|
| 461 |
+
if "message" in data and isinstance(data["message"], dict):
|
| 462 |
+
final_message = data["message"] # type: ignore
|
| 463 |
+
verbose_logger.debug("Found final message")
|
| 464 |
+
|
| 465 |
+
# Process event data
|
| 466 |
+
if "event" in data and isinstance(data["event"], dict):
|
| 467 |
+
event_payload = data["event"]
|
| 468 |
+
verbose_logger.debug(
|
| 469 |
+
f"Event payload keys: {list(event_payload.keys())}"
|
| 470 |
+
)
|
| 471 |
+
|
| 472 |
+
# Extract usage metadata
|
| 473 |
+
if usage := self._extract_usage_from_event(data):
|
| 474 |
+
usage_data = usage
|
| 475 |
+
verbose_logger.debug(f"Found usage data: {usage_data}")
|
| 476 |
+
|
| 477 |
+
# Collect content deltas
|
| 478 |
+
if text := self._extract_content_delta(data):
|
| 479 |
+
content_blocks.append(text)
|
| 480 |
+
|
| 481 |
+
# Build final content
|
| 482 |
+
content = (
|
| 483 |
+
self._extract_content_from_message(final_message)
|
| 484 |
+
if final_message
|
| 485 |
+
else "".join(content_blocks)
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
verbose_logger.debug(f"Final usage_data: {usage_data}")
|
| 489 |
+
|
| 490 |
+
return AgentCoreParsedResponse(
|
| 491 |
+
content=content, usage=usage_data, final_message=final_message
|
| 492 |
+
)
|
| 493 |
+
|
| 494 |
+
def _stream_agentcore_response_sync(
|
| 495 |
+
self,
|
| 496 |
+
response: httpx.Response,
|
| 497 |
+
model: str,
|
| 498 |
+
):
|
| 499 |
+
"""
|
| 500 |
+
Internal sync generator that parses SSE and yields ModelResponse chunks.
|
| 501 |
+
"""
|
| 502 |
+
buffer = ""
|
| 503 |
+
for text_chunk in response.iter_text():
|
| 504 |
+
buffer += text_chunk
|
| 505 |
+
|
| 506 |
+
# Process complete lines
|
| 507 |
+
while "\n" in buffer:
|
| 508 |
+
line, buffer = buffer.split("\n", 1)
|
| 509 |
+
line = line.strip()
|
| 510 |
+
|
| 511 |
+
if not line or not line.startswith("data:"):
|
| 512 |
+
continue
|
| 513 |
+
|
| 514 |
+
json_str = line[5:].strip()
|
| 515 |
+
if not json_str:
|
| 516 |
+
continue
|
| 517 |
+
|
| 518 |
+
try:
|
| 519 |
+
data_obj = json.loads(json_str)
|
| 520 |
+
if not isinstance(data_obj, dict):
|
| 521 |
+
continue
|
| 522 |
+
|
| 523 |
+
# Process contentBlockDelta events
|
| 524 |
+
if "event" in data_obj and isinstance(data_obj["event"], dict):
|
| 525 |
+
event_payload = data_obj["event"]
|
| 526 |
+
content_block_delta = event_payload.get("contentBlockDelta")
|
| 527 |
+
|
| 528 |
+
if content_block_delta:
|
| 529 |
+
delta = content_block_delta.get("delta", {})
|
| 530 |
+
text = delta.get("text", "")
|
| 531 |
+
|
| 532 |
+
if text:
|
| 533 |
+
chunk = ModelResponseStream(
|
| 534 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 535 |
+
created=0,
|
| 536 |
+
model=model,
|
| 537 |
+
object="chat.completion.chunk",
|
| 538 |
+
)
|
| 539 |
+
chunk.choices = [
|
| 540 |
+
StreamingChoices(
|
| 541 |
+
finish_reason=None,
|
| 542 |
+
index=0,
|
| 543 |
+
delta=Delta(content=text, role="assistant"),
|
| 544 |
+
)
|
| 545 |
+
]
|
| 546 |
+
yield chunk
|
| 547 |
+
|
| 548 |
+
# Process metadata/usage
|
| 549 |
+
metadata = event_payload.get("metadata")
|
| 550 |
+
if metadata and "usage" in metadata:
|
| 551 |
+
chunk = ModelResponseStream(
|
| 552 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 553 |
+
created=0,
|
| 554 |
+
model=model,
|
| 555 |
+
object="chat.completion.chunk",
|
| 556 |
+
)
|
| 557 |
+
chunk.choices = [
|
| 558 |
+
StreamingChoices(
|
| 559 |
+
finish_reason="stop",
|
| 560 |
+
index=0,
|
| 561 |
+
delta=Delta(),
|
| 562 |
+
)
|
| 563 |
+
]
|
| 564 |
+
usage_data: AgentCoreUsage = metadata["usage"] # type: ignore
|
| 565 |
+
setattr(
|
| 566 |
+
chunk,
|
| 567 |
+
"usage",
|
| 568 |
+
Usage(
|
| 569 |
+
prompt_tokens=usage_data.get("inputTokens", 0),
|
| 570 |
+
completion_tokens=usage_data.get("outputTokens", 0),
|
| 571 |
+
total_tokens=usage_data.get("totalTokens", 0),
|
| 572 |
+
),
|
| 573 |
+
)
|
| 574 |
+
yield chunk
|
| 575 |
+
|
| 576 |
+
# Process final message
|
| 577 |
+
if "message" in data_obj and isinstance(data_obj["message"], dict):
|
| 578 |
+
chunk = ModelResponseStream(
|
| 579 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 580 |
+
created=0,
|
| 581 |
+
model=model,
|
| 582 |
+
object="chat.completion.chunk",
|
| 583 |
+
)
|
| 584 |
+
chunk.choices = [
|
| 585 |
+
StreamingChoices(
|
| 586 |
+
finish_reason="stop",
|
| 587 |
+
index=0,
|
| 588 |
+
delta=Delta(),
|
| 589 |
+
)
|
| 590 |
+
]
|
| 591 |
+
yield chunk
|
| 592 |
+
|
| 593 |
+
except json.JSONDecodeError:
|
| 594 |
+
verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
|
| 595 |
+
continue
|
| 596 |
+
|
| 597 |
+
def get_sync_custom_stream_wrapper(
|
| 598 |
+
self,
|
| 599 |
+
model: str,
|
| 600 |
+
custom_llm_provider: str,
|
| 601 |
+
logging_obj: LiteLLMLoggingObj,
|
| 602 |
+
api_base: str,
|
| 603 |
+
headers: dict,
|
| 604 |
+
data: dict,
|
| 605 |
+
messages: list,
|
| 606 |
+
client: Optional[Union[HTTPHandler, "AsyncHTTPHandler"]] = None,
|
| 607 |
+
json_mode: Optional[bool] = None,
|
| 608 |
+
signed_json_body: Optional[bytes] = None,
|
| 609 |
+
) -> "CustomStreamWrapper":
|
| 610 |
+
"""
|
| 611 |
+
Simplified sync streaming - returns a generator that yields ModelResponse chunks.
|
| 612 |
+
"""
|
| 613 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 614 |
+
HTTPHandler,
|
| 615 |
+
_get_httpx_client,
|
| 616 |
+
)
|
| 617 |
+
|
| 618 |
+
if client is None or not isinstance(client, HTTPHandler):
|
| 619 |
+
client = _get_httpx_client(params={})
|
| 620 |
+
|
| 621 |
+
verbose_logger.debug(f"Making sync streaming request to: {api_base}")
|
| 622 |
+
|
| 623 |
+
# Make streaming request
|
| 624 |
+
response = client.post(
|
| 625 |
+
api_base,
|
| 626 |
+
headers=headers,
|
| 627 |
+
data=signed_json_body if signed_json_body else json.dumps(data),
|
| 628 |
+
stream=True,
|
| 629 |
+
logging_obj=logging_obj,
|
| 630 |
+
)
|
| 631 |
+
|
| 632 |
+
if response.status_code != 200:
|
| 633 |
+
raise BedrockError(
|
| 634 |
+
status_code=response.status_code, message=str(response.read())
|
| 635 |
+
)
|
| 636 |
+
|
| 637 |
+
# LOGGING
|
| 638 |
+
logging_obj.post_call(
|
| 639 |
+
input=messages,
|
| 640 |
+
api_key="",
|
| 641 |
+
original_response="first stream response received",
|
| 642 |
+
additional_args={"complete_input_dict": data},
|
| 643 |
+
)
|
| 644 |
+
|
| 645 |
+
# Check if response is JSON (agent used sync return) instead of SSE
|
| 646 |
+
content_type = response.headers.get("content-type", "").lower()
|
| 647 |
+
if "application/json" in content_type:
|
| 648 |
+
verbose_logger.debug(
|
| 649 |
+
"AgentCore streaming: received JSON response instead of SSE, "
|
| 650 |
+
"converting to single-chunk stream"
|
| 651 |
+
)
|
| 652 |
+
try:
|
| 653 |
+
body = response.read()
|
| 654 |
+
response_json = json.loads(body)
|
| 655 |
+
except (json.JSONDecodeError, Exception) as e:
|
| 656 |
+
raise BedrockError(
|
| 657 |
+
status_code=response.status_code,
|
| 658 |
+
message=f"AgentCore: Failed to read/parse JSON response body: {e}",
|
| 659 |
+
)
|
| 660 |
+
parsed = self._parse_json_response(response_json)
|
| 661 |
+
|
| 662 |
+
def _json_as_sync_stream():
|
| 663 |
+
# Content chunk
|
| 664 |
+
content_chunk = ModelResponseStream(
|
| 665 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 666 |
+
created=0,
|
| 667 |
+
model=model,
|
| 668 |
+
object="chat.completion.chunk",
|
| 669 |
+
)
|
| 670 |
+
content_chunk.choices = [
|
| 671 |
+
StreamingChoices(
|
| 672 |
+
finish_reason=None,
|
| 673 |
+
index=0,
|
| 674 |
+
delta=Delta(content=parsed["content"], role="assistant"),
|
| 675 |
+
)
|
| 676 |
+
]
|
| 677 |
+
yield content_chunk
|
| 678 |
+
|
| 679 |
+
# Stop sentinel chunk (matches SSE path convention)
|
| 680 |
+
stop_chunk = ModelResponseStream(
|
| 681 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 682 |
+
created=0,
|
| 683 |
+
model=model,
|
| 684 |
+
object="chat.completion.chunk",
|
| 685 |
+
)
|
| 686 |
+
stop_chunk.choices = [
|
| 687 |
+
StreamingChoices(
|
| 688 |
+
finish_reason="stop",
|
| 689 |
+
index=0,
|
| 690 |
+
delta=Delta(),
|
| 691 |
+
)
|
| 692 |
+
]
|
| 693 |
+
yield stop_chunk
|
| 694 |
+
|
| 695 |
+
return CustomStreamWrapper(
|
| 696 |
+
completion_stream=_json_as_sync_stream(),
|
| 697 |
+
model=model,
|
| 698 |
+
custom_llm_provider="bedrock",
|
| 699 |
+
logging_obj=logging_obj,
|
| 700 |
+
)
|
| 701 |
+
|
| 702 |
+
# SSE stream (text/event-stream or default) - use existing SSE parser
|
| 703 |
+
return CustomStreamWrapper(
|
| 704 |
+
completion_stream=self._stream_agentcore_response_sync(response, model),
|
| 705 |
+
model=model,
|
| 706 |
+
custom_llm_provider="bedrock",
|
| 707 |
+
logging_obj=logging_obj,
|
| 708 |
+
)
|
| 709 |
+
|
| 710 |
+
async def _stream_agentcore_response(
|
| 711 |
+
self,
|
| 712 |
+
response: httpx.Response,
|
| 713 |
+
model: str,
|
| 714 |
+
) -> AsyncGenerator[ModelResponseStream, None]:
|
| 715 |
+
"""
|
| 716 |
+
Internal async generator that parses SSE and yields ModelResponse chunks.
|
| 717 |
+
"""
|
| 718 |
+
buffer = ""
|
| 719 |
+
async for text_chunk in response.aiter_text():
|
| 720 |
+
buffer += text_chunk
|
| 721 |
+
|
| 722 |
+
# Process complete lines
|
| 723 |
+
while "\n" in buffer:
|
| 724 |
+
line, buffer = buffer.split("\n", 1)
|
| 725 |
+
line = line.strip()
|
| 726 |
+
|
| 727 |
+
if not line or not line.startswith("data:"):
|
| 728 |
+
continue
|
| 729 |
+
|
| 730 |
+
json_str = line[5:].strip()
|
| 731 |
+
if not json_str:
|
| 732 |
+
continue
|
| 733 |
+
|
| 734 |
+
try:
|
| 735 |
+
data_obj = json.loads(json_str)
|
| 736 |
+
if not isinstance(data_obj, dict):
|
| 737 |
+
continue
|
| 738 |
+
|
| 739 |
+
# Process contentBlockDelta events
|
| 740 |
+
if "event" in data_obj and isinstance(data_obj["event"], dict):
|
| 741 |
+
event_payload = data_obj["event"]
|
| 742 |
+
content_block_delta = event_payload.get("contentBlockDelta")
|
| 743 |
+
|
| 744 |
+
if content_block_delta:
|
| 745 |
+
delta = content_block_delta.get("delta", {})
|
| 746 |
+
text = delta.get("text", "")
|
| 747 |
+
|
| 748 |
+
if text:
|
| 749 |
+
chunk = ModelResponseStream(
|
| 750 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 751 |
+
created=0,
|
| 752 |
+
model=model,
|
| 753 |
+
object="chat.completion.chunk",
|
| 754 |
+
)
|
| 755 |
+
chunk.choices = [
|
| 756 |
+
StreamingChoices(
|
| 757 |
+
finish_reason=None,
|
| 758 |
+
index=0,
|
| 759 |
+
delta=Delta(content=text, role="assistant"),
|
| 760 |
+
)
|
| 761 |
+
]
|
| 762 |
+
yield chunk
|
| 763 |
+
|
| 764 |
+
# Process metadata/usage
|
| 765 |
+
metadata = event_payload.get("metadata")
|
| 766 |
+
if metadata and "usage" in metadata:
|
| 767 |
+
chunk = ModelResponseStream(
|
| 768 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 769 |
+
created=0,
|
| 770 |
+
model=model,
|
| 771 |
+
object="chat.completion.chunk",
|
| 772 |
+
)
|
| 773 |
+
chunk.choices = [
|
| 774 |
+
StreamingChoices(
|
| 775 |
+
finish_reason="stop",
|
| 776 |
+
index=0,
|
| 777 |
+
delta=Delta(),
|
| 778 |
+
)
|
| 779 |
+
]
|
| 780 |
+
usage_data: AgentCoreUsage = metadata["usage"] # type: ignore
|
| 781 |
+
setattr(
|
| 782 |
+
chunk,
|
| 783 |
+
"usage",
|
| 784 |
+
Usage(
|
| 785 |
+
prompt_tokens=usage_data.get("inputTokens", 0),
|
| 786 |
+
completion_tokens=usage_data.get("outputTokens", 0),
|
| 787 |
+
total_tokens=usage_data.get("totalTokens", 0),
|
| 788 |
+
),
|
| 789 |
+
)
|
| 790 |
+
yield chunk
|
| 791 |
+
|
| 792 |
+
# Process final message
|
| 793 |
+
if "message" in data_obj and isinstance(data_obj["message"], dict):
|
| 794 |
+
chunk = ModelResponseStream(
|
| 795 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 796 |
+
created=0,
|
| 797 |
+
model=model,
|
| 798 |
+
object="chat.completion.chunk",
|
| 799 |
+
)
|
| 800 |
+
chunk.choices = [
|
| 801 |
+
StreamingChoices(
|
| 802 |
+
finish_reason="stop",
|
| 803 |
+
index=0,
|
| 804 |
+
delta=Delta(),
|
| 805 |
+
)
|
| 806 |
+
]
|
| 807 |
+
yield chunk
|
| 808 |
+
|
| 809 |
+
except json.JSONDecodeError:
|
| 810 |
+
verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
|
| 811 |
+
continue
|
| 812 |
+
|
| 813 |
+
async def get_async_custom_stream_wrapper(
|
| 814 |
+
self,
|
| 815 |
+
model: str,
|
| 816 |
+
custom_llm_provider: str,
|
| 817 |
+
logging_obj: LiteLLMLoggingObj,
|
| 818 |
+
api_base: str,
|
| 819 |
+
headers: dict,
|
| 820 |
+
data: dict,
|
| 821 |
+
messages: list,
|
| 822 |
+
client: Optional["AsyncHTTPHandler"] = None,
|
| 823 |
+
json_mode: Optional[bool] = None,
|
| 824 |
+
signed_json_body: Optional[bytes] = None,
|
| 825 |
+
) -> "CustomStreamWrapper":
|
| 826 |
+
"""
|
| 827 |
+
Simplified async streaming - returns an async generator that yields ModelResponse chunks.
|
| 828 |
+
"""
|
| 829 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 830 |
+
AsyncHTTPHandler,
|
| 831 |
+
get_async_httpx_client,
|
| 832 |
+
)
|
| 833 |
+
|
| 834 |
+
if client is None or not isinstance(client, AsyncHTTPHandler):
|
| 835 |
+
client = get_async_httpx_client(
|
| 836 |
+
llm_provider=cast(Any, "bedrock"), params={}
|
| 837 |
+
)
|
| 838 |
+
|
| 839 |
+
verbose_logger.debug(f"Making async streaming request to: {api_base}")
|
| 840 |
+
|
| 841 |
+
# Make async streaming request
|
| 842 |
+
response = await client.post(
|
| 843 |
+
api_base,
|
| 844 |
+
headers=headers,
|
| 845 |
+
data=signed_json_body if signed_json_body else json.dumps(data),
|
| 846 |
+
stream=True,
|
| 847 |
+
logging_obj=logging_obj,
|
| 848 |
+
)
|
| 849 |
+
|
| 850 |
+
if response.status_code != 200:
|
| 851 |
+
raise BedrockError(
|
| 852 |
+
status_code=response.status_code, message=str(await response.aread())
|
| 853 |
+
)
|
| 854 |
+
|
| 855 |
+
# LOGGING
|
| 856 |
+
logging_obj.post_call(
|
| 857 |
+
input=messages,
|
| 858 |
+
api_key="",
|
| 859 |
+
original_response="first stream response received",
|
| 860 |
+
additional_args={"complete_input_dict": data},
|
| 861 |
+
)
|
| 862 |
+
|
| 863 |
+
# Check if response is JSON (agent used sync return) instead of SSE
|
| 864 |
+
content_type = response.headers.get("content-type", "").lower()
|
| 865 |
+
if "application/json" in content_type:
|
| 866 |
+
verbose_logger.debug(
|
| 867 |
+
"AgentCore streaming: received JSON response instead of SSE, "
|
| 868 |
+
"converting to single-chunk stream"
|
| 869 |
+
)
|
| 870 |
+
try:
|
| 871 |
+
body = await response.aread()
|
| 872 |
+
response_json = json.loads(body)
|
| 873 |
+
except (json.JSONDecodeError, Exception) as e:
|
| 874 |
+
raise BedrockError(
|
| 875 |
+
status_code=response.status_code,
|
| 876 |
+
message=f"AgentCore: Failed to read/parse JSON response body: {e}",
|
| 877 |
+
)
|
| 878 |
+
parsed = self._parse_json_response(response_json)
|
| 879 |
+
|
| 880 |
+
async def _json_as_async_stream() -> AsyncGenerator[
|
| 881 |
+
ModelResponseStream, None
|
| 882 |
+
]:
|
| 883 |
+
# Content chunk
|
| 884 |
+
content_chunk = ModelResponseStream(
|
| 885 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 886 |
+
created=0,
|
| 887 |
+
model=model,
|
| 888 |
+
object="chat.completion.chunk",
|
| 889 |
+
)
|
| 890 |
+
content_chunk.choices = [
|
| 891 |
+
StreamingChoices(
|
| 892 |
+
finish_reason=None,
|
| 893 |
+
index=0,
|
| 894 |
+
delta=Delta(content=parsed["content"], role="assistant"),
|
| 895 |
+
)
|
| 896 |
+
]
|
| 897 |
+
yield content_chunk
|
| 898 |
+
|
| 899 |
+
# Stop sentinel chunk (matches SSE path convention)
|
| 900 |
+
stop_chunk = ModelResponseStream(
|
| 901 |
+
id=f"chatcmpl-{uuid.uuid4()}",
|
| 902 |
+
created=0,
|
| 903 |
+
model=model,
|
| 904 |
+
object="chat.completion.chunk",
|
| 905 |
+
)
|
| 906 |
+
stop_chunk.choices = [
|
| 907 |
+
StreamingChoices(
|
| 908 |
+
finish_reason="stop",
|
| 909 |
+
index=0,
|
| 910 |
+
delta=Delta(),
|
| 911 |
+
)
|
| 912 |
+
]
|
| 913 |
+
yield stop_chunk
|
| 914 |
+
|
| 915 |
+
return CustomStreamWrapper(
|
| 916 |
+
completion_stream=_json_as_async_stream(),
|
| 917 |
+
model=model,
|
| 918 |
+
custom_llm_provider="bedrock",
|
| 919 |
+
logging_obj=logging_obj,
|
| 920 |
+
)
|
| 921 |
+
|
| 922 |
+
# SSE stream (text/event-stream or default) - use existing SSE parser
|
| 923 |
+
return CustomStreamWrapper(
|
| 924 |
+
completion_stream=self._stream_agentcore_response(response, model),
|
| 925 |
+
model=model,
|
| 926 |
+
custom_llm_provider="bedrock",
|
| 927 |
+
logging_obj=logging_obj,
|
| 928 |
+
)
|
| 929 |
+
|
| 930 |
+
@property
|
| 931 |
+
def has_custom_stream_wrapper(self) -> bool:
|
| 932 |
+
"""Indicates that this config has custom streaming support."""
|
| 933 |
+
return True
|
| 934 |
+
|
| 935 |
+
@property
|
| 936 |
+
def supports_stream_param_in_request_body(self) -> bool:
|
| 937 |
+
"""
|
| 938 |
+
AgentCore does not allow passing `stream` in the request body.
|
| 939 |
+
Streaming is automatic based on the response format.
|
| 940 |
+
"""
|
| 941 |
+
return False
|
| 942 |
+
|
| 943 |
+
def transform_response(
|
| 944 |
+
self,
|
| 945 |
+
model: str,
|
| 946 |
+
raw_response: httpx.Response,
|
| 947 |
+
model_response: ModelResponse,
|
| 948 |
+
logging_obj: LiteLLMLoggingObj,
|
| 949 |
+
request_data: dict,
|
| 950 |
+
messages: List[AllMessageValues],
|
| 951 |
+
optional_params: dict,
|
| 952 |
+
litellm_params: dict,
|
| 953 |
+
encoding: Any,
|
| 954 |
+
api_key: Optional[str] = None,
|
| 955 |
+
json_mode: Optional[bool] = None,
|
| 956 |
+
) -> ModelResponse:
|
| 957 |
+
"""
|
| 958 |
+
Transform the AgentCore response to LiteLLM ModelResponse format.
|
| 959 |
+
AgentCore can return either JSON or SSE (Server-Sent Events) stream responses.
|
| 960 |
+
|
| 961 |
+
Note: For streaming responses, use get_streaming_response() instead.
|
| 962 |
+
"""
|
| 963 |
+
try:
|
| 964 |
+
# Parse the response based on content type (JSON or SSE)
|
| 965 |
+
parsed_data = self._get_parsed_response(raw_response)
|
| 966 |
+
|
| 967 |
+
content = parsed_data["content"]
|
| 968 |
+
usage_data = parsed_data["usage"]
|
| 969 |
+
|
| 970 |
+
verbose_logger.debug(f"Parsed content length: {len(content)}")
|
| 971 |
+
verbose_logger.debug(f"Usage data: {usage_data}")
|
| 972 |
+
|
| 973 |
+
# Create the message
|
| 974 |
+
message = Message(content=content, role="assistant")
|
| 975 |
+
|
| 976 |
+
# Create choices
|
| 977 |
+
choice = Choices(finish_reason="stop", index=0, message=message)
|
| 978 |
+
|
| 979 |
+
# Update model response
|
| 980 |
+
model_response.choices = [choice]
|
| 981 |
+
model_response.model = model
|
| 982 |
+
|
| 983 |
+
# Add usage information if available
|
| 984 |
+
# Note: AgentCore JSON responses don't include usage data
|
| 985 |
+
# SSE responses may include usage in metadata events
|
| 986 |
+
if usage_data:
|
| 987 |
+
usage = Usage(
|
| 988 |
+
prompt_tokens=usage_data.get("inputTokens", 0),
|
| 989 |
+
completion_tokens=usage_data.get("outputTokens", 0),
|
| 990 |
+
total_tokens=usage_data.get("totalTokens", 0),
|
| 991 |
+
)
|
| 992 |
+
setattr(model_response, "usage", usage)
|
| 993 |
+
else:
|
| 994 |
+
# Calculate token usage using LiteLLM's token counter
|
| 995 |
+
verbose_logger.debug(
|
| 996 |
+
"No usage data from AgentCore - calculating tokens"
|
| 997 |
+
)
|
| 998 |
+
calculated_usage = self._calculate_usage(model, messages, content)
|
| 999 |
+
if calculated_usage:
|
| 1000 |
+
setattr(model_response, "usage", calculated_usage)
|
| 1001 |
+
|
| 1002 |
+
return model_response
|
| 1003 |
+
|
| 1004 |
+
except Exception as e:
|
| 1005 |
+
verbose_logger.error(
|
| 1006 |
+
f"Error processing Bedrock AgentCore response: {str(e)}"
|
| 1007 |
+
)
|
| 1008 |
+
raise BedrockError(
|
| 1009 |
+
message=f"Error processing response: {str(e)}",
|
| 1010 |
+
status_code=raw_response.status_code,
|
| 1011 |
+
)
|
| 1012 |
+
|
| 1013 |
+
def validate_environment(
|
| 1014 |
+
self,
|
| 1015 |
+
headers: dict,
|
| 1016 |
+
model: str,
|
| 1017 |
+
messages: List[AllMessageValues],
|
| 1018 |
+
optional_params: dict,
|
| 1019 |
+
litellm_params: dict,
|
| 1020 |
+
api_key: Optional[str] = None,
|
| 1021 |
+
api_base: Optional[str] = None,
|
| 1022 |
+
) -> dict:
|
| 1023 |
+
return headers
|
| 1024 |
+
|
| 1025 |
+
def get_error_class(
|
| 1026 |
+
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
| 1027 |
+
) -> BaseLLMException:
|
| 1028 |
+
return BedrockError(status_code=status_code, message=error_message)
|
| 1029 |
+
|
| 1030 |
+
def should_fake_stream(
|
| 1031 |
+
self,
|
| 1032 |
+
model: Optional[str],
|
| 1033 |
+
stream: Optional[bool],
|
| 1034 |
+
custom_llm_provider: Optional[str] = None,
|
| 1035 |
+
) -> bool:
|
| 1036 |
+
# AgentCore supports true streaming - don't buffer
|
| 1037 |
+
return False
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_handler.py
ADDED
|
@@ -0,0 +1,512 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from typing import Any, Optional, Union
|
| 3 |
+
|
| 4 |
+
import httpx
|
| 5 |
+
|
| 6 |
+
import litellm
|
| 7 |
+
from litellm.anthropic_beta_headers_manager import (
|
| 8 |
+
update_headers_with_filtered_beta,
|
| 9 |
+
)
|
| 10 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
| 11 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 12 |
+
AsyncHTTPHandler,
|
| 13 |
+
HTTPHandler,
|
| 14 |
+
_get_httpx_client,
|
| 15 |
+
get_async_httpx_client,
|
| 16 |
+
)
|
| 17 |
+
from litellm.types.utils import ModelResponse
|
| 18 |
+
from litellm.utils import CustomStreamWrapper
|
| 19 |
+
|
| 20 |
+
from ..base_aws_llm import BaseAWSLLM, Credentials
|
| 21 |
+
from ..common_utils import BedrockError, _get_all_bedrock_regions
|
| 22 |
+
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def make_sync_call(
|
| 26 |
+
client: Optional[HTTPHandler],
|
| 27 |
+
api_base: str,
|
| 28 |
+
headers: dict,
|
| 29 |
+
data: str,
|
| 30 |
+
model: str,
|
| 31 |
+
messages: list,
|
| 32 |
+
logging_obj: LiteLLMLoggingObject,
|
| 33 |
+
json_mode: Optional[bool] = False,
|
| 34 |
+
fake_stream: bool = False,
|
| 35 |
+
stream_chunk_size: int = 1024,
|
| 36 |
+
):
|
| 37 |
+
if client is None:
|
| 38 |
+
client = _get_httpx_client() # Create a new client if none provided
|
| 39 |
+
|
| 40 |
+
response = client.post(
|
| 41 |
+
api_base,
|
| 42 |
+
headers=headers,
|
| 43 |
+
data=data,
|
| 44 |
+
stream=not fake_stream,
|
| 45 |
+
logging_obj=logging_obj,
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
if response.status_code != 200:
|
| 49 |
+
raise BedrockError(
|
| 50 |
+
status_code=response.status_code, message=str(response.read())
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
if fake_stream:
|
| 54 |
+
model_response: (
|
| 55 |
+
ModelResponse
|
| 56 |
+
) = litellm.AmazonConverseConfig()._transform_response(
|
| 57 |
+
model=model,
|
| 58 |
+
response=response,
|
| 59 |
+
model_response=litellm.ModelResponse(),
|
| 60 |
+
stream=True,
|
| 61 |
+
logging_obj=logging_obj,
|
| 62 |
+
optional_params={},
|
| 63 |
+
api_key="",
|
| 64 |
+
data=data,
|
| 65 |
+
messages=messages,
|
| 66 |
+
encoding=litellm.encoding,
|
| 67 |
+
) # type: ignore
|
| 68 |
+
completion_stream: Any = MockResponseIterator(
|
| 69 |
+
model_response=model_response, json_mode=json_mode
|
| 70 |
+
)
|
| 71 |
+
else:
|
| 72 |
+
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
|
| 73 |
+
completion_stream = decoder.iter_bytes(
|
| 74 |
+
response.iter_bytes(chunk_size=stream_chunk_size)
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
# LOGGING
|
| 78 |
+
logging_obj.post_call(
|
| 79 |
+
input=messages,
|
| 80 |
+
api_key="",
|
| 81 |
+
original_response="first stream response received",
|
| 82 |
+
additional_args={"complete_input_dict": data},
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
return completion_stream
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class BedrockConverseLLM(BaseAWSLLM):
|
| 89 |
+
def __init__(self) -> None:
|
| 90 |
+
super().__init__()
|
| 91 |
+
|
| 92 |
+
async def async_streaming(
|
| 93 |
+
self,
|
| 94 |
+
model: str,
|
| 95 |
+
messages: list,
|
| 96 |
+
api_base: str,
|
| 97 |
+
model_response: ModelResponse,
|
| 98 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 99 |
+
encoding,
|
| 100 |
+
logging_obj,
|
| 101 |
+
stream,
|
| 102 |
+
optional_params: dict,
|
| 103 |
+
litellm_params: dict,
|
| 104 |
+
credentials: Credentials,
|
| 105 |
+
logger_fn=None,
|
| 106 |
+
headers={},
|
| 107 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 108 |
+
fake_stream: bool = False,
|
| 109 |
+
json_mode: Optional[bool] = False,
|
| 110 |
+
api_key: Optional[str] = None,
|
| 111 |
+
stream_chunk_size: int = 1024,
|
| 112 |
+
) -> CustomStreamWrapper:
|
| 113 |
+
request_data = await litellm.AmazonConverseConfig()._async_transform_request(
|
| 114 |
+
model=model,
|
| 115 |
+
messages=messages,
|
| 116 |
+
optional_params=optional_params,
|
| 117 |
+
litellm_params=litellm_params,
|
| 118 |
+
headers=headers,
|
| 119 |
+
)
|
| 120 |
+
data = json.dumps(request_data)
|
| 121 |
+
|
| 122 |
+
prepped = self.get_request_headers(
|
| 123 |
+
credentials=credentials,
|
| 124 |
+
aws_region_name=litellm_params.get("aws_region_name") or "us-west-2",
|
| 125 |
+
extra_headers=headers,
|
| 126 |
+
endpoint_url=api_base,
|
| 127 |
+
data=data,
|
| 128 |
+
headers=headers,
|
| 129 |
+
api_key=api_key,
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
## LOGGING
|
| 133 |
+
logging_obj.pre_call(
|
| 134 |
+
input=messages,
|
| 135 |
+
api_key="",
|
| 136 |
+
additional_args={
|
| 137 |
+
"complete_input_dict": data,
|
| 138 |
+
"api_base": api_base,
|
| 139 |
+
"headers": dict(prepped.headers),
|
| 140 |
+
},
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
completion_stream = await make_call(
|
| 144 |
+
client=client,
|
| 145 |
+
api_base=api_base,
|
| 146 |
+
headers=dict(prepped.headers),
|
| 147 |
+
data=data,
|
| 148 |
+
model=model,
|
| 149 |
+
messages=messages,
|
| 150 |
+
logging_obj=logging_obj,
|
| 151 |
+
fake_stream=fake_stream,
|
| 152 |
+
json_mode=json_mode,
|
| 153 |
+
stream_chunk_size=stream_chunk_size,
|
| 154 |
+
)
|
| 155 |
+
streaming_response = CustomStreamWrapper(
|
| 156 |
+
completion_stream=completion_stream,
|
| 157 |
+
model=model,
|
| 158 |
+
custom_llm_provider="bedrock",
|
| 159 |
+
logging_obj=logging_obj,
|
| 160 |
+
)
|
| 161 |
+
return streaming_response
|
| 162 |
+
|
| 163 |
+
async def async_completion(
|
| 164 |
+
self,
|
| 165 |
+
model: str,
|
| 166 |
+
messages: list,
|
| 167 |
+
api_base: str,
|
| 168 |
+
model_response: ModelResponse,
|
| 169 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 170 |
+
encoding,
|
| 171 |
+
logging_obj: LiteLLMLoggingObject,
|
| 172 |
+
stream,
|
| 173 |
+
optional_params: dict,
|
| 174 |
+
litellm_params: dict,
|
| 175 |
+
credentials: Credentials,
|
| 176 |
+
logger_fn=None,
|
| 177 |
+
headers: dict = {},
|
| 178 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 179 |
+
api_key: Optional[str] = None,
|
| 180 |
+
) -> Union[ModelResponse, CustomStreamWrapper]:
|
| 181 |
+
request_data = await litellm.AmazonConverseConfig()._async_transform_request(
|
| 182 |
+
model=model,
|
| 183 |
+
messages=messages,
|
| 184 |
+
optional_params=optional_params,
|
| 185 |
+
litellm_params=litellm_params,
|
| 186 |
+
headers=headers,
|
| 187 |
+
)
|
| 188 |
+
data = json.dumps(request_data)
|
| 189 |
+
|
| 190 |
+
prepped = self.get_request_headers(
|
| 191 |
+
credentials=credentials,
|
| 192 |
+
aws_region_name=litellm_params.get("aws_region_name") or "us-west-2",
|
| 193 |
+
extra_headers=headers,
|
| 194 |
+
endpoint_url=api_base,
|
| 195 |
+
data=data,
|
| 196 |
+
headers=headers,
|
| 197 |
+
api_key=api_key,
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
## LOGGING
|
| 201 |
+
logging_obj.pre_call(
|
| 202 |
+
input=messages,
|
| 203 |
+
api_key="",
|
| 204 |
+
additional_args={
|
| 205 |
+
"complete_input_dict": data,
|
| 206 |
+
"api_base": api_base,
|
| 207 |
+
"headers": prepped.headers,
|
| 208 |
+
},
|
| 209 |
+
)
|
| 210 |
+
|
| 211 |
+
headers = dict(prepped.headers)
|
| 212 |
+
if client is None or not isinstance(client, AsyncHTTPHandler):
|
| 213 |
+
_params = {}
|
| 214 |
+
if timeout is not None:
|
| 215 |
+
if isinstance(timeout, float) or isinstance(timeout, int):
|
| 216 |
+
timeout = httpx.Timeout(timeout)
|
| 217 |
+
_params["timeout"] = timeout
|
| 218 |
+
client = get_async_httpx_client(
|
| 219 |
+
params=_params, llm_provider=litellm.LlmProviders.BEDROCK
|
| 220 |
+
)
|
| 221 |
+
else:
|
| 222 |
+
client = client # type: ignore
|
| 223 |
+
|
| 224 |
+
try:
|
| 225 |
+
response = await client.post(
|
| 226 |
+
url=api_base,
|
| 227 |
+
headers=headers,
|
| 228 |
+
data=data,
|
| 229 |
+
logging_obj=logging_obj,
|
| 230 |
+
) # type: ignore
|
| 231 |
+
response.raise_for_status()
|
| 232 |
+
except httpx.HTTPStatusError as err:
|
| 233 |
+
error_code = err.response.status_code
|
| 234 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 235 |
+
except httpx.TimeoutException:
|
| 236 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 237 |
+
|
| 238 |
+
return litellm.AmazonConverseConfig()._transform_response(
|
| 239 |
+
model=model,
|
| 240 |
+
response=response,
|
| 241 |
+
model_response=model_response,
|
| 242 |
+
stream=stream if isinstance(stream, bool) else False,
|
| 243 |
+
logging_obj=logging_obj,
|
| 244 |
+
api_key="",
|
| 245 |
+
data=data,
|
| 246 |
+
messages=messages,
|
| 247 |
+
optional_params=optional_params,
|
| 248 |
+
encoding=encoding,
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
def completion( # noqa: PLR0915
|
| 252 |
+
self,
|
| 253 |
+
model: str,
|
| 254 |
+
messages: list,
|
| 255 |
+
api_base: Optional[str],
|
| 256 |
+
custom_prompt_dict: dict,
|
| 257 |
+
model_response: ModelResponse,
|
| 258 |
+
encoding,
|
| 259 |
+
logging_obj: LiteLLMLoggingObject,
|
| 260 |
+
optional_params: dict,
|
| 261 |
+
acompletion: bool,
|
| 262 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 263 |
+
litellm_params: dict,
|
| 264 |
+
logger_fn=None,
|
| 265 |
+
extra_headers: Optional[dict] = None,
|
| 266 |
+
client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None,
|
| 267 |
+
api_key: Optional[str] = None,
|
| 268 |
+
):
|
| 269 |
+
## SETUP ##
|
| 270 |
+
stream = optional_params.pop("stream", None)
|
| 271 |
+
stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
|
| 272 |
+
unencoded_model_id = optional_params.pop("model_id", None)
|
| 273 |
+
fake_stream = optional_params.pop("fake_stream", False)
|
| 274 |
+
json_mode = optional_params.get("json_mode", False)
|
| 275 |
+
if unencoded_model_id is not None:
|
| 276 |
+
modelId = self.encode_model_id(model_id=unencoded_model_id)
|
| 277 |
+
else:
|
| 278 |
+
# Strip nova spec prefixes before encoding model ID for API URL
|
| 279 |
+
_model_for_id = model
|
| 280 |
+
_stripped = _model_for_id
|
| 281 |
+
for rp in ["bedrock/converse/", "bedrock/", "converse/"]:
|
| 282 |
+
if _stripped.startswith(rp):
|
| 283 |
+
_stripped = _stripped[len(rp) :]
|
| 284 |
+
break
|
| 285 |
+
# Strip embedded region prefix (e.g. "bedrock/us-east-1/model" -> "model")
|
| 286 |
+
# and capture it so it can be used as aws_region_name below.
|
| 287 |
+
_region_from_model: Optional[str] = None
|
| 288 |
+
_potential_region = _stripped.split("/", 1)[0]
|
| 289 |
+
if _potential_region in _get_all_bedrock_regions() and "/" in _stripped:
|
| 290 |
+
_region_from_model = _potential_region
|
| 291 |
+
_stripped = _stripped.split("/", 1)[1]
|
| 292 |
+
_model_for_id = _stripped
|
| 293 |
+
for _nova_prefix in ["nova-2/", "nova/"]:
|
| 294 |
+
if _stripped.startswith(_nova_prefix):
|
| 295 |
+
_model_for_id = _model_for_id.replace(_nova_prefix, "", 1)
|
| 296 |
+
break
|
| 297 |
+
modelId = self.encode_model_id(model_id=_model_for_id)
|
| 298 |
+
# Inject region extracted from model path so _get_aws_region_name picks it up
|
| 299 |
+
if (
|
| 300 |
+
_region_from_model is not None
|
| 301 |
+
and "aws_region_name" not in optional_params
|
| 302 |
+
):
|
| 303 |
+
optional_params["aws_region_name"] = _region_from_model
|
| 304 |
+
|
| 305 |
+
fake_stream = litellm.AmazonConverseConfig().should_fake_stream(
|
| 306 |
+
fake_stream=fake_stream,
|
| 307 |
+
model=model,
|
| 308 |
+
stream=stream,
|
| 309 |
+
custom_llm_provider="bedrock",
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
### SET REGION NAME ###
|
| 313 |
+
aws_region_name = self._get_aws_region_name(
|
| 314 |
+
optional_params=optional_params,
|
| 315 |
+
model=model,
|
| 316 |
+
model_id=unencoded_model_id,
|
| 317 |
+
)
|
| 318 |
+
|
| 319 |
+
## CREDENTIALS ##
|
| 320 |
+
# pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them
|
| 321 |
+
aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
|
| 322 |
+
aws_access_key_id = optional_params.pop("aws_access_key_id", None)
|
| 323 |
+
aws_session_token = optional_params.pop("aws_session_token", None)
|
| 324 |
+
aws_role_name = optional_params.pop("aws_role_name", None)
|
| 325 |
+
aws_session_name = optional_params.pop("aws_session_name", None)
|
| 326 |
+
aws_profile_name = optional_params.pop("aws_profile_name", None)
|
| 327 |
+
aws_bedrock_runtime_endpoint = optional_params.pop(
|
| 328 |
+
"aws_bedrock_runtime_endpoint", None
|
| 329 |
+
) # https://bedrock-runtime.{region_name}.amazonaws.com
|
| 330 |
+
aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
|
| 331 |
+
aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
|
| 332 |
+
aws_external_id = optional_params.pop("aws_external_id", None)
|
| 333 |
+
optional_params.pop("aws_region_name", None)
|
| 334 |
+
|
| 335 |
+
litellm_params[
|
| 336 |
+
"aws_region_name"
|
| 337 |
+
] = aws_region_name # [DO NOT DELETE] important for async calls
|
| 338 |
+
|
| 339 |
+
credentials: Credentials = self.get_credentials(
|
| 340 |
+
aws_access_key_id=aws_access_key_id,
|
| 341 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 342 |
+
aws_session_token=aws_session_token,
|
| 343 |
+
aws_region_name=aws_region_name,
|
| 344 |
+
aws_session_name=aws_session_name,
|
| 345 |
+
aws_profile_name=aws_profile_name,
|
| 346 |
+
aws_role_name=aws_role_name,
|
| 347 |
+
aws_web_identity_token=aws_web_identity_token,
|
| 348 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 349 |
+
aws_external_id=aws_external_id,
|
| 350 |
+
)
|
| 351 |
+
|
| 352 |
+
### SET RUNTIME ENDPOINT ###
|
| 353 |
+
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
|
| 354 |
+
api_base=api_base,
|
| 355 |
+
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
| 356 |
+
aws_region_name=aws_region_name,
|
| 357 |
+
)
|
| 358 |
+
if (stream is not None and stream is True) and not fake_stream:
|
| 359 |
+
endpoint_url = f"{endpoint_url}/model/{modelId}/converse-stream"
|
| 360 |
+
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse-stream"
|
| 361 |
+
else:
|
| 362 |
+
endpoint_url = f"{endpoint_url}/model/{modelId}/converse"
|
| 363 |
+
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse"
|
| 364 |
+
|
| 365 |
+
## COMPLETION CALL
|
| 366 |
+
headers = {"Content-Type": "application/json"}
|
| 367 |
+
if extra_headers is not None:
|
| 368 |
+
headers = {"Content-Type": "application/json", **extra_headers}
|
| 369 |
+
|
| 370 |
+
# Filter beta headers in HTTP headers before making the request
|
| 371 |
+
headers = update_headers_with_filtered_beta(
|
| 372 |
+
headers=headers, provider="bedrock_converse"
|
| 373 |
+
)
|
| 374 |
+
### ROUTING (ASYNC, STREAMING, SYNC)
|
| 375 |
+
if acompletion:
|
| 376 |
+
if isinstance(client, HTTPHandler):
|
| 377 |
+
client = None
|
| 378 |
+
if stream is True:
|
| 379 |
+
return self.async_streaming(
|
| 380 |
+
model=model,
|
| 381 |
+
messages=messages,
|
| 382 |
+
api_base=proxy_endpoint_url,
|
| 383 |
+
model_response=model_response,
|
| 384 |
+
encoding=encoding,
|
| 385 |
+
logging_obj=logging_obj,
|
| 386 |
+
optional_params=optional_params,
|
| 387 |
+
stream=True,
|
| 388 |
+
litellm_params=litellm_params,
|
| 389 |
+
logger_fn=logger_fn,
|
| 390 |
+
headers=headers,
|
| 391 |
+
timeout=timeout,
|
| 392 |
+
client=client,
|
| 393 |
+
json_mode=json_mode,
|
| 394 |
+
fake_stream=fake_stream,
|
| 395 |
+
credentials=credentials,
|
| 396 |
+
api_key=api_key,
|
| 397 |
+
stream_chunk_size=stream_chunk_size,
|
| 398 |
+
) # type: ignore
|
| 399 |
+
### ASYNC COMPLETION
|
| 400 |
+
return self.async_completion(
|
| 401 |
+
model=model,
|
| 402 |
+
messages=messages,
|
| 403 |
+
api_base=proxy_endpoint_url,
|
| 404 |
+
model_response=model_response,
|
| 405 |
+
encoding=encoding,
|
| 406 |
+
logging_obj=logging_obj,
|
| 407 |
+
optional_params=optional_params,
|
| 408 |
+
stream=stream, # type: ignore
|
| 409 |
+
litellm_params=litellm_params,
|
| 410 |
+
logger_fn=logger_fn,
|
| 411 |
+
headers=headers,
|
| 412 |
+
timeout=timeout,
|
| 413 |
+
client=client,
|
| 414 |
+
credentials=credentials,
|
| 415 |
+
api_key=api_key,
|
| 416 |
+
) # type: ignore
|
| 417 |
+
|
| 418 |
+
## TRANSFORMATION ##
|
| 419 |
+
|
| 420 |
+
_data = litellm.AmazonConverseConfig()._transform_request(
|
| 421 |
+
model=model,
|
| 422 |
+
messages=messages,
|
| 423 |
+
optional_params=optional_params,
|
| 424 |
+
litellm_params=litellm_params,
|
| 425 |
+
headers=extra_headers,
|
| 426 |
+
)
|
| 427 |
+
data = json.dumps(_data)
|
| 428 |
+
|
| 429 |
+
prepped = self.get_request_headers(
|
| 430 |
+
credentials=credentials,
|
| 431 |
+
aws_region_name=aws_region_name,
|
| 432 |
+
extra_headers=extra_headers,
|
| 433 |
+
endpoint_url=proxy_endpoint_url,
|
| 434 |
+
data=data,
|
| 435 |
+
headers=headers,
|
| 436 |
+
api_key=api_key,
|
| 437 |
+
)
|
| 438 |
+
|
| 439 |
+
## LOGGING
|
| 440 |
+
logging_obj.pre_call(
|
| 441 |
+
input=messages,
|
| 442 |
+
api_key="",
|
| 443 |
+
additional_args={
|
| 444 |
+
"complete_input_dict": data,
|
| 445 |
+
"api_base": proxy_endpoint_url,
|
| 446 |
+
"headers": prepped.headers,
|
| 447 |
+
},
|
| 448 |
+
)
|
| 449 |
+
if client is None or isinstance(client, AsyncHTTPHandler):
|
| 450 |
+
_params = {}
|
| 451 |
+
if timeout is not None:
|
| 452 |
+
if isinstance(timeout, float) or isinstance(timeout, int):
|
| 453 |
+
timeout = httpx.Timeout(timeout)
|
| 454 |
+
_params["timeout"] = timeout
|
| 455 |
+
client = _get_httpx_client(_params) # type: ignore
|
| 456 |
+
else:
|
| 457 |
+
client = client
|
| 458 |
+
|
| 459 |
+
if stream is not None and stream is True:
|
| 460 |
+
completion_stream = make_sync_call(
|
| 461 |
+
client=(
|
| 462 |
+
client
|
| 463 |
+
if client is not None and isinstance(client, HTTPHandler)
|
| 464 |
+
else None
|
| 465 |
+
),
|
| 466 |
+
api_base=proxy_endpoint_url,
|
| 467 |
+
headers=prepped.headers, # type: ignore
|
| 468 |
+
data=data,
|
| 469 |
+
model=model,
|
| 470 |
+
messages=messages,
|
| 471 |
+
logging_obj=logging_obj,
|
| 472 |
+
json_mode=json_mode,
|
| 473 |
+
fake_stream=fake_stream,
|
| 474 |
+
stream_chunk_size=stream_chunk_size,
|
| 475 |
+
)
|
| 476 |
+
streaming_response = CustomStreamWrapper(
|
| 477 |
+
completion_stream=completion_stream,
|
| 478 |
+
model=model,
|
| 479 |
+
custom_llm_provider="bedrock",
|
| 480 |
+
logging_obj=logging_obj,
|
| 481 |
+
)
|
| 482 |
+
|
| 483 |
+
return streaming_response
|
| 484 |
+
|
| 485 |
+
### COMPLETION
|
| 486 |
+
|
| 487 |
+
try:
|
| 488 |
+
response = client.post(
|
| 489 |
+
url=proxy_endpoint_url,
|
| 490 |
+
headers=prepped.headers,
|
| 491 |
+
data=data,
|
| 492 |
+
logging_obj=logging_obj,
|
| 493 |
+
) # type: ignore
|
| 494 |
+
response.raise_for_status()
|
| 495 |
+
except httpx.HTTPStatusError as err:
|
| 496 |
+
error_code = err.response.status_code
|
| 497 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 498 |
+
except httpx.TimeoutException:
|
| 499 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 500 |
+
|
| 501 |
+
return litellm.AmazonConverseConfig()._transform_response(
|
| 502 |
+
model=model,
|
| 503 |
+
response=response,
|
| 504 |
+
model_response=model_response,
|
| 505 |
+
stream=stream if isinstance(stream, bool) else False,
|
| 506 |
+
logging_obj=logging_obj,
|
| 507 |
+
api_key="",
|
| 508 |
+
data=data,
|
| 509 |
+
messages=messages,
|
| 510 |
+
optional_params=optional_params,
|
| 511 |
+
encoding=encoding,
|
| 512 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/handler.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Uses base_llm_http_handler to call the 'converse like' endpoint.
|
| 3 |
+
|
| 4 |
+
Relevant issue: https://github.com/BerriAI/litellm/issues/8085
|
| 5 |
+
"""
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/transformation.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Uses `converse_transformation.py` to transform the messages to the format required by Bedrock Converse.
|
| 3 |
+
"""
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_transformation.py
ADDED
|
@@ -0,0 +1,2126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Translating between OpenAI's `/chat/completion` format and Amazon's `/converse` format
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import copy
|
| 6 |
+
import json
|
| 7 |
+
import time
|
| 8 |
+
import types
|
| 9 |
+
from typing import List, Literal, Optional, Tuple, Union, cast, overload
|
| 10 |
+
|
| 11 |
+
import httpx
|
| 12 |
+
|
| 13 |
+
import litellm
|
| 14 |
+
from litellm._logging import verbose_logger
|
| 15 |
+
from litellm.constants import (
|
| 16 |
+
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
|
| 17 |
+
RESPONSE_FORMAT_TOOL_NAME,
|
| 18 |
+
)
|
| 19 |
+
from litellm.litellm_core_utils.core_helpers import (
|
| 20 |
+
filter_exceptions_from_params,
|
| 21 |
+
filter_internal_params,
|
| 22 |
+
map_finish_reason,
|
| 23 |
+
safe_deep_copy,
|
| 24 |
+
)
|
| 25 |
+
from litellm.litellm_core_utils.litellm_logging import Logging
|
| 26 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 27 |
+
_parse_content_for_reasoning,
|
| 28 |
+
)
|
| 29 |
+
from litellm.litellm_core_utils.prompt_templates.factory import (
|
| 30 |
+
BedrockConverseMessagesProcessor,
|
| 31 |
+
_bedrock_converse_messages_pt,
|
| 32 |
+
_bedrock_tools_pt,
|
| 33 |
+
)
|
| 34 |
+
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
| 35 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
| 36 |
+
from litellm.types.llms.bedrock import *
|
| 37 |
+
from litellm.types.llms.openai import (
|
| 38 |
+
AllMessageValues,
|
| 39 |
+
ChatCompletionAssistantMessage,
|
| 40 |
+
ChatCompletionRedactedThinkingBlock,
|
| 41 |
+
ChatCompletionResponseMessage,
|
| 42 |
+
ChatCompletionSystemMessage,
|
| 43 |
+
ChatCompletionThinkingBlock,
|
| 44 |
+
ChatCompletionToolCallChunk,
|
| 45 |
+
ChatCompletionToolCallFunctionChunk,
|
| 46 |
+
ChatCompletionToolParam,
|
| 47 |
+
ChatCompletionToolParamFunctionChunk,
|
| 48 |
+
ChatCompletionUserMessage,
|
| 49 |
+
OpenAIChatCompletionToolParam,
|
| 50 |
+
OpenAIMessageContentListBlock,
|
| 51 |
+
)
|
| 52 |
+
from litellm.types.utils import (
|
| 53 |
+
ChatCompletionMessageToolCall,
|
| 54 |
+
CompletionTokensDetailsWrapper,
|
| 55 |
+
Function,
|
| 56 |
+
Message,
|
| 57 |
+
ModelResponse,
|
| 58 |
+
PromptTokensDetailsWrapper,
|
| 59 |
+
Usage,
|
| 60 |
+
)
|
| 61 |
+
from litellm.utils import (
|
| 62 |
+
add_dummy_tool,
|
| 63 |
+
any_assistant_message_has_thinking_blocks,
|
| 64 |
+
has_tool_call_blocks,
|
| 65 |
+
last_assistant_with_tool_calls_has_no_thinking_blocks,
|
| 66 |
+
supports_reasoning,
|
| 67 |
+
token_counter,
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
from ..common_utils import (
|
| 71 |
+
BedrockError,
|
| 72 |
+
BedrockModelInfo,
|
| 73 |
+
get_anthropic_beta_from_headers,
|
| 74 |
+
get_bedrock_tool_name,
|
| 75 |
+
is_claude_4_5_on_bedrock,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
# Computer use tool prefixes supported by Bedrock
|
| 79 |
+
BEDROCK_COMPUTER_USE_TOOLS = [
|
| 80 |
+
"computer_use_preview",
|
| 81 |
+
"computer_",
|
| 82 |
+
"bash_",
|
| 83 |
+
"text_editor_",
|
| 84 |
+
]
|
| 85 |
+
|
| 86 |
+
# Beta header patterns that are not supported by Bedrock Converse API
|
| 87 |
+
# These will be filtered out to prevent errors
|
| 88 |
+
UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS = [
|
| 89 |
+
"advanced-tool-use", # Bedrock Converse doesn't support advanced-tool-use beta headers
|
| 90 |
+
"prompt-caching", # Prompt caching not supported in Converse API
|
| 91 |
+
"compact-2026-01-12", # The compact beta feature is not currently supported on the Converse and ConverseStream APIs
|
| 92 |
+
]
|
| 93 |
+
|
| 94 |
+
# Models that support Bedrock's native structured outputs API (outputConfig.textFormat)
|
| 95 |
+
# Uses substring matching against the Bedrock model ID
|
| 96 |
+
# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
|
| 97 |
+
BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS = {
|
| 98 |
+
# Anthropic Claude 4.5+
|
| 99 |
+
"claude-haiku-4-5",
|
| 100 |
+
"claude-sonnet-4-5",
|
| 101 |
+
"claude-opus-4-5",
|
| 102 |
+
"claude-opus-4-6",
|
| 103 |
+
# Qwen3
|
| 104 |
+
"qwen3",
|
| 105 |
+
# DeepSeek
|
| 106 |
+
"deepseek-v3.1",
|
| 107 |
+
# Gemma 3
|
| 108 |
+
"gemma-3",
|
| 109 |
+
# MiniMax
|
| 110 |
+
"minimax-m2",
|
| 111 |
+
# Mistral (magistral-small excluded: broken constrained decoding on Bedrock)
|
| 112 |
+
"ministral",
|
| 113 |
+
"mistral-large-3",
|
| 114 |
+
"voxtral",
|
| 115 |
+
# Moonshot
|
| 116 |
+
"kimi-k2",
|
| 117 |
+
# NVIDIA
|
| 118 |
+
"nemotron-nano",
|
| 119 |
+
# OpenAI (gpt-oss excluded: broken constrained decoding, works via tool-call fallback)
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
class AmazonConverseConfig(BaseConfig):
|
| 124 |
+
"""
|
| 125 |
+
Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
|
| 126 |
+
#2 - https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html#conversation-inference-supported-models-features
|
| 127 |
+
"""
|
| 128 |
+
|
| 129 |
+
maxTokens: Optional[int]
|
| 130 |
+
stopSequences: Optional[List[str]]
|
| 131 |
+
temperature: Optional[int]
|
| 132 |
+
topP: Optional[int]
|
| 133 |
+
topK: Optional[int]
|
| 134 |
+
|
| 135 |
+
def __init__(
|
| 136 |
+
self,
|
| 137 |
+
maxTokens: Optional[int] = None,
|
| 138 |
+
stopSequences: Optional[List[str]] = None,
|
| 139 |
+
temperature: Optional[int] = None,
|
| 140 |
+
topP: Optional[int] = None,
|
| 141 |
+
topK: Optional[int] = None,
|
| 142 |
+
) -> None:
|
| 143 |
+
locals_ = locals().copy()
|
| 144 |
+
for key, value in locals_.items():
|
| 145 |
+
if key != "self" and value is not None:
|
| 146 |
+
setattr(self.__class__, key, value)
|
| 147 |
+
|
| 148 |
+
@property
|
| 149 |
+
def custom_llm_provider(self) -> Optional[str]:
|
| 150 |
+
return "bedrock_converse"
|
| 151 |
+
|
| 152 |
+
@classmethod
|
| 153 |
+
def get_config_blocks(cls) -> dict:
|
| 154 |
+
return {
|
| 155 |
+
"guardrailConfig": GuardrailConfigBlock,
|
| 156 |
+
"performanceConfig": PerformanceConfigBlock,
|
| 157 |
+
"serviceTier": ServiceTierBlock,
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
@staticmethod
|
| 161 |
+
def _convert_consecutive_user_messages_to_guarded_text(
|
| 162 |
+
messages: List[AllMessageValues], optional_params: dict
|
| 163 |
+
) -> List[AllMessageValues]:
|
| 164 |
+
"""
|
| 165 |
+
Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present
|
| 166 |
+
and no guarded_text is already present in those messages.
|
| 167 |
+
"""
|
| 168 |
+
# Check if guardrailConfig is present
|
| 169 |
+
if "guardrailConfig" not in optional_params:
|
| 170 |
+
return messages
|
| 171 |
+
|
| 172 |
+
# Find all consecutive user messages at the end
|
| 173 |
+
consecutive_user_message_indices = []
|
| 174 |
+
for i in range(len(messages) - 1, -1, -1):
|
| 175 |
+
if messages[i].get("role") == "user":
|
| 176 |
+
consecutive_user_message_indices.append(i)
|
| 177 |
+
else:
|
| 178 |
+
break
|
| 179 |
+
|
| 180 |
+
if not consecutive_user_message_indices:
|
| 181 |
+
return messages
|
| 182 |
+
|
| 183 |
+
# Process each consecutive user message
|
| 184 |
+
messages_copy = copy.deepcopy(messages)
|
| 185 |
+
for user_message_index in consecutive_user_message_indices:
|
| 186 |
+
user_message = messages_copy[user_message_index]
|
| 187 |
+
content = user_message.get("content", [])
|
| 188 |
+
|
| 189 |
+
if isinstance(content, list):
|
| 190 |
+
has_guarded_text = any(
|
| 191 |
+
isinstance(item, dict) and item.get("type") == "guarded_text"
|
| 192 |
+
for item in content
|
| 193 |
+
)
|
| 194 |
+
if has_guarded_text:
|
| 195 |
+
continue # Skip this message if it already has guarded_text
|
| 196 |
+
|
| 197 |
+
# Convert text elements to guarded_text
|
| 198 |
+
new_content = []
|
| 199 |
+
for item in content:
|
| 200 |
+
if isinstance(item, dict) and item.get("type") == "text":
|
| 201 |
+
new_item = {"type": "guarded_text", "text": item["text"]} # type: ignore
|
| 202 |
+
new_content.append(new_item)
|
| 203 |
+
else:
|
| 204 |
+
new_content.append(item)
|
| 205 |
+
|
| 206 |
+
messages_copy[user_message_index]["content"] = new_content # type: ignore
|
| 207 |
+
elif isinstance(content, str):
|
| 208 |
+
# If content is a string, convert it to guarded_text
|
| 209 |
+
messages_copy[user_message_index]["content"] = [ # type: ignore
|
| 210 |
+
{"type": "guarded_text", "text": content} # type: ignore
|
| 211 |
+
]
|
| 212 |
+
|
| 213 |
+
return messages_copy
|
| 214 |
+
|
| 215 |
+
@classmethod
|
| 216 |
+
def get_config(cls):
|
| 217 |
+
return {
|
| 218 |
+
k: v
|
| 219 |
+
for k, v in cls.__dict__.items()
|
| 220 |
+
if not k.startswith("__")
|
| 221 |
+
and not isinstance(
|
| 222 |
+
v,
|
| 223 |
+
(
|
| 224 |
+
types.FunctionType,
|
| 225 |
+
types.BuiltinFunctionType,
|
| 226 |
+
classmethod,
|
| 227 |
+
staticmethod,
|
| 228 |
+
),
|
| 229 |
+
)
|
| 230 |
+
and v is not None
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
def _validate_request_metadata(self, metadata: dict) -> None:
|
| 234 |
+
"""
|
| 235 |
+
Validate requestMetadata according to AWS Bedrock Converse API constraints.
|
| 236 |
+
|
| 237 |
+
Constraints:
|
| 238 |
+
- Maximum of 16 items
|
| 239 |
+
- Keys: 1-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{1,256}
|
| 240 |
+
- Values: 0-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{0,256}
|
| 241 |
+
"""
|
| 242 |
+
import re
|
| 243 |
+
|
| 244 |
+
if not isinstance(metadata, dict):
|
| 245 |
+
raise litellm.exceptions.BadRequestError(
|
| 246 |
+
message="requestMetadata must be a dictionary",
|
| 247 |
+
model="bedrock",
|
| 248 |
+
llm_provider="bedrock",
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
if len(metadata) > 16:
|
| 252 |
+
raise litellm.exceptions.BadRequestError(
|
| 253 |
+
message="requestMetadata can contain a maximum of 16 items",
|
| 254 |
+
model="bedrock",
|
| 255 |
+
llm_provider="bedrock",
|
| 256 |
+
)
|
| 257 |
+
|
| 258 |
+
key_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
|
| 259 |
+
value_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
|
| 260 |
+
|
| 261 |
+
for key, value in metadata.items():
|
| 262 |
+
if not isinstance(key, str):
|
| 263 |
+
raise litellm.exceptions.BadRequestError(
|
| 264 |
+
message="requestMetadata keys must be strings",
|
| 265 |
+
model="bedrock",
|
| 266 |
+
llm_provider="bedrock",
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
if not isinstance(value, str):
|
| 270 |
+
raise litellm.exceptions.BadRequestError(
|
| 271 |
+
message="requestMetadata values must be strings",
|
| 272 |
+
model="bedrock",
|
| 273 |
+
llm_provider="bedrock",
|
| 274 |
+
)
|
| 275 |
+
|
| 276 |
+
if len(key) == 0 or len(key) > 256:
|
| 277 |
+
raise litellm.exceptions.BadRequestError(
|
| 278 |
+
message="requestMetadata key length must be 1-256 characters",
|
| 279 |
+
model="bedrock",
|
| 280 |
+
llm_provider="bedrock",
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
if len(value) > 256:
|
| 284 |
+
raise litellm.exceptions.BadRequestError(
|
| 285 |
+
message="requestMetadata value length must be 0-256 characters",
|
| 286 |
+
model="bedrock",
|
| 287 |
+
llm_provider="bedrock",
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
if not key_pattern.match(key):
|
| 291 |
+
raise litellm.exceptions.BadRequestError(
|
| 292 |
+
message=f"requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
|
| 293 |
+
model="bedrock",
|
| 294 |
+
llm_provider="bedrock",
|
| 295 |
+
)
|
| 296 |
+
|
| 297 |
+
if not value_pattern.match(value):
|
| 298 |
+
raise litellm.exceptions.BadRequestError(
|
| 299 |
+
message=f"requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
|
| 300 |
+
model="bedrock",
|
| 301 |
+
llm_provider="bedrock",
|
| 302 |
+
)
|
| 303 |
+
|
| 304 |
+
def _is_nova_2_model(self, model: str) -> bool:
|
| 305 |
+
"""
|
| 306 |
+
Check if the model is a Nova 2 model that supports reasoningConfig.
|
| 307 |
+
|
| 308 |
+
Nova 2 models use a different reasoning configuration structure compared to
|
| 309 |
+
Anthropic's thinking parameter and GPT-OSS's reasoning_effort parameter.
|
| 310 |
+
|
| 311 |
+
Supported models:
|
| 312 |
+
- amazon.nova-2-lite-v1:0
|
| 313 |
+
- amazon.nova-2-pro-preview-20251202-v1:0
|
| 314 |
+
- us.amazon.nova-2-lite-v1:0
|
| 315 |
+
- eu.amazon.nova-2-lite-v1:0
|
| 316 |
+
- apac.amazon.nova-2-lite-v1:0
|
| 317 |
+
- (and other regional variants)
|
| 318 |
+
|
| 319 |
+
Args:
|
| 320 |
+
model: The model identifier
|
| 321 |
+
|
| 322 |
+
Returns:
|
| 323 |
+
True if the model is a Nova 2 model, False otherwise
|
| 324 |
+
|
| 325 |
+
Examples:
|
| 326 |
+
>>> config = AmazonConverseConfig()
|
| 327 |
+
>>> config._is_nova_2_model("amazon.nova-2-lite-v1:0")
|
| 328 |
+
True
|
| 329 |
+
>>> config._is_nova_2_model("us.amazon.nova-2-lite-v1:0")
|
| 330 |
+
True
|
| 331 |
+
>>> config._is_nova_2_model("us.amazon.nova-2-pro-preview-20251202-v1:0")
|
| 332 |
+
True
|
| 333 |
+
>>> config._is_nova_2_model("amazon.nova-pro-1-5-v1:0")
|
| 334 |
+
False
|
| 335 |
+
>>> config._is_nova_2_model("amazon.nova-pro-v1:0")
|
| 336 |
+
False
|
| 337 |
+
"""
|
| 338 |
+
# Remove provider routing prefix if present (bedrock/converse/, bedrock/, converse/)
|
| 339 |
+
model_without_region = model
|
| 340 |
+
for routing_prefix in ["bedrock/converse/", "bedrock/", "converse/"]:
|
| 341 |
+
if model_without_region.startswith(routing_prefix):
|
| 342 |
+
model_without_region = model_without_region[len(routing_prefix) :]
|
| 343 |
+
break
|
| 344 |
+
|
| 345 |
+
# Remove regional prefix if present (us., eu., apac.)
|
| 346 |
+
for prefix in ["us.", "eu.", "apac."]:
|
| 347 |
+
if model_without_region.startswith(prefix):
|
| 348 |
+
model_without_region = model_without_region[len(prefix) :]
|
| 349 |
+
break
|
| 350 |
+
|
| 351 |
+
# Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.)
|
| 352 |
+
# Also check for nova-2/ spec prefix for imported models
|
| 353 |
+
return model_without_region.startswith(
|
| 354 |
+
"amazon.nova-2-"
|
| 355 |
+
) or model_without_region.startswith("nova-2/")
|
| 356 |
+
|
| 357 |
+
def _map_web_search_options(
|
| 358 |
+
self, web_search_options: dict, model: str
|
| 359 |
+
) -> Optional[BedrockToolBlock]:
|
| 360 |
+
"""
|
| 361 |
+
Map web_search_options to Nova grounding systemTool.
|
| 362 |
+
|
| 363 |
+
Nova grounding (web search) is only supported on Amazon Nova models.
|
| 364 |
+
Returns None for non-Nova models.
|
| 365 |
+
|
| 366 |
+
Args:
|
| 367 |
+
web_search_options: The web_search_options dict from the request
|
| 368 |
+
model: The model identifier string
|
| 369 |
+
|
| 370 |
+
Returns:
|
| 371 |
+
BedrockToolBlock with systemTool for Nova models, None otherwise
|
| 372 |
+
|
| 373 |
+
Reference: https://docs.aws.amazon.com/nova/latest/userguide/grounding.html
|
| 374 |
+
"""
|
| 375 |
+
# Only Nova models support nova_grounding
|
| 376 |
+
# Model strings can be like: "amazon.nova-pro-v1:0", "us.amazon.nova-pro-v1:0", etc.
|
| 377 |
+
if "nova" not in model.lower():
|
| 378 |
+
verbose_logger.debug(
|
| 379 |
+
f"web_search_options passed but model {model} is not a Nova model. "
|
| 380 |
+
"Nova grounding is only supported on Amazon Nova models."
|
| 381 |
+
)
|
| 382 |
+
return None
|
| 383 |
+
|
| 384 |
+
# Nova doesn't support search_context_size or user_location params
|
| 385 |
+
# (unlike Anthropic), so we just enable grounding with no options
|
| 386 |
+
return BedrockToolBlock(systemTool={"name": "nova_grounding"})
|
| 387 |
+
|
| 388 |
+
def _transform_reasoning_effort_to_reasoning_config(
|
| 389 |
+
self, reasoning_effort: str
|
| 390 |
+
) -> dict:
|
| 391 |
+
"""
|
| 392 |
+
Transform reasoning_effort parameter to Nova 2 reasoningConfig structure.
|
| 393 |
+
|
| 394 |
+
Nova 2 models use a reasoningConfig structure in additionalModelRequestFields
|
| 395 |
+
that differs from both Anthropic's thinking parameter and GPT-OSS's reasoning_effort.
|
| 396 |
+
|
| 397 |
+
Args:
|
| 398 |
+
reasoning_effort: The reasoning effort level, must be "low" or "high"
|
| 399 |
+
|
| 400 |
+
Returns:
|
| 401 |
+
dict: A dictionary containing the reasoningConfig structure:
|
| 402 |
+
{
|
| 403 |
+
"reasoningConfig": {
|
| 404 |
+
"type": "enabled",
|
| 405 |
+
"maxReasoningEffort": "low" | "medium" |"high"
|
| 406 |
+
}
|
| 407 |
+
}
|
| 408 |
+
|
| 409 |
+
Raises:
|
| 410 |
+
BadRequestError: If reasoning_effort is not "low", "medium" or "high"
|
| 411 |
+
|
| 412 |
+
Examples:
|
| 413 |
+
>>> config = AmazonConverseConfig()
|
| 414 |
+
>>> config._transform_reasoning_effort_to_reasoning_config("high")
|
| 415 |
+
{'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}}
|
| 416 |
+
>>> config._transform_reasoning_effort_to_reasoning_config("low")
|
| 417 |
+
{'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'low'}}
|
| 418 |
+
"""
|
| 419 |
+
valid_values = ["low", "medium", "high"]
|
| 420 |
+
if reasoning_effort not in valid_values:
|
| 421 |
+
raise litellm.exceptions.BadRequestError(
|
| 422 |
+
message=f"Invalid reasoning_effort value '{reasoning_effort}' for Nova 2 models. "
|
| 423 |
+
f"Supported values: {valid_values}",
|
| 424 |
+
model="amazon.nova-2-lite-v1:0",
|
| 425 |
+
llm_provider="bedrock_converse",
|
| 426 |
+
)
|
| 427 |
+
|
| 428 |
+
return {
|
| 429 |
+
"reasoningConfig": {
|
| 430 |
+
"type": "enabled",
|
| 431 |
+
"maxReasoningEffort": reasoning_effort,
|
| 432 |
+
}
|
| 433 |
+
}
|
| 434 |
+
|
| 435 |
+
def _handle_reasoning_effort_parameter(
|
| 436 |
+
self, model: str, reasoning_effort: str, optional_params: dict
|
| 437 |
+
) -> None:
|
| 438 |
+
"""
|
| 439 |
+
Handle the reasoning_effort parameter based on the model type.
|
| 440 |
+
|
| 441 |
+
Different model families handle reasoning effort differently:
|
| 442 |
+
- GPT-OSS models: Keep reasoning_effort as-is (passed to additionalModelRequestFields)
|
| 443 |
+
- Nova 2 models: Transform to reasoningConfig structure
|
| 444 |
+
- Other models (Anthropic, etc.): Convert to thinking parameter
|
| 445 |
+
|
| 446 |
+
Args:
|
| 447 |
+
model: The model identifier
|
| 448 |
+
reasoning_effort: The reasoning effort value
|
| 449 |
+
optional_params: Dictionary of optional parameters to update in-place
|
| 450 |
+
|
| 451 |
+
Examples:
|
| 452 |
+
>>> config = AmazonConverseConfig()
|
| 453 |
+
>>> params = {}
|
| 454 |
+
>>> config._handle_reasoning_effort_parameter("gpt-oss-model", "high", params)
|
| 455 |
+
>>> params
|
| 456 |
+
{'reasoning_effort': 'high'}
|
| 457 |
+
|
| 458 |
+
>>> params = {}
|
| 459 |
+
>>> config._handle_reasoning_effort_parameter("amazon.nova-2-lite-v1:0", "high", params)
|
| 460 |
+
>>> params
|
| 461 |
+
{'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}}
|
| 462 |
+
|
| 463 |
+
>>> params = {}
|
| 464 |
+
>>> config._handle_reasoning_effort_parameter("anthropic.claude-3", "high", params)
|
| 465 |
+
>>> params
|
| 466 |
+
{'thinking': {'type': 'enabled', 'budget_tokens': 10000}}
|
| 467 |
+
"""
|
| 468 |
+
if "gpt-oss" in model:
|
| 469 |
+
# GPT-OSS models: keep reasoning_effort as-is
|
| 470 |
+
# It will be passed through to additionalModelRequestFields
|
| 471 |
+
optional_params["reasoning_effort"] = reasoning_effort
|
| 472 |
+
elif self._is_nova_2_model(model):
|
| 473 |
+
# Nova 2 models: transform to reasoningConfig
|
| 474 |
+
reasoning_config = self._transform_reasoning_effort_to_reasoning_config(
|
| 475 |
+
reasoning_effort
|
| 476 |
+
)
|
| 477 |
+
optional_params.update(reasoning_config)
|
| 478 |
+
else:
|
| 479 |
+
# Anthropic and other models: convert to thinking parameter
|
| 480 |
+
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
|
| 481 |
+
reasoning_effort=reasoning_effort, model=model
|
| 482 |
+
)
|
| 483 |
+
|
| 484 |
+
@staticmethod
|
| 485 |
+
def _clamp_thinking_budget_tokens(optional_params: dict) -> None:
|
| 486 |
+
"""
|
| 487 |
+
Clamp thinking.budget_tokens to the Bedrock minimum (1024).
|
| 488 |
+
|
| 489 |
+
Bedrock returns a 400 error if budget_tokens < 1024.
|
| 490 |
+
"""
|
| 491 |
+
thinking = optional_params.get("thinking")
|
| 492 |
+
if isinstance(thinking, dict):
|
| 493 |
+
budget = thinking.get("budget_tokens")
|
| 494 |
+
if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS:
|
| 495 |
+
verbose_logger.debug(
|
| 496 |
+
"Bedrock requires thinking.budget_tokens >= %d, got %d. "
|
| 497 |
+
"Clamping to minimum.",
|
| 498 |
+
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
|
| 499 |
+
budget,
|
| 500 |
+
)
|
| 501 |
+
thinking["budget_tokens"] = BEDROCK_MIN_THINKING_BUDGET_TOKENS
|
| 502 |
+
|
| 503 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 504 |
+
from litellm.utils import supports_function_calling
|
| 505 |
+
|
| 506 |
+
supported_params = [
|
| 507 |
+
"max_tokens",
|
| 508 |
+
"max_completion_tokens",
|
| 509 |
+
"stream",
|
| 510 |
+
"stream_options",
|
| 511 |
+
"stop",
|
| 512 |
+
"temperature",
|
| 513 |
+
"top_p",
|
| 514 |
+
"extra_headers",
|
| 515 |
+
"response_format",
|
| 516 |
+
"requestMetadata",
|
| 517 |
+
"service_tier",
|
| 518 |
+
"parallel_tool_calls",
|
| 519 |
+
]
|
| 520 |
+
|
| 521 |
+
if (
|
| 522 |
+
"arn" in model
|
| 523 |
+
): # we can't infer the model from the arn, so just add all params
|
| 524 |
+
supported_params.append("tools")
|
| 525 |
+
supported_params.append("tool_choice")
|
| 526 |
+
supported_params.append("thinking")
|
| 527 |
+
supported_params.append("reasoning_effort")
|
| 528 |
+
# For nova imported models, also add web_search_options
|
| 529 |
+
if "nova" in model.lower():
|
| 530 |
+
supported_params.append("web_search_options")
|
| 531 |
+
return supported_params
|
| 532 |
+
|
| 533 |
+
## Filter out 'cross-region' from model name
|
| 534 |
+
base_model = BedrockModelInfo.get_base_model(model)
|
| 535 |
+
|
| 536 |
+
if (
|
| 537 |
+
base_model.startswith("anthropic")
|
| 538 |
+
or base_model.startswith("mistral")
|
| 539 |
+
or base_model.startswith("cohere")
|
| 540 |
+
or base_model.startswith("meta.llama3-1")
|
| 541 |
+
or base_model.startswith("meta.llama3-2")
|
| 542 |
+
or base_model.startswith("meta.llama3-3")
|
| 543 |
+
or base_model.startswith("meta.llama4")
|
| 544 |
+
or base_model.startswith("amazon.nova")
|
| 545 |
+
or supports_function_calling(
|
| 546 |
+
model=model, custom_llm_provider=self.custom_llm_provider
|
| 547 |
+
)
|
| 548 |
+
):
|
| 549 |
+
supported_params.append("tools")
|
| 550 |
+
|
| 551 |
+
# Nova models support web_search_options (mapped to nova_grounding systemTool)
|
| 552 |
+
if base_model.startswith("amazon.nova"):
|
| 553 |
+
supported_params.append("web_search_options")
|
| 554 |
+
|
| 555 |
+
if litellm.utils.supports_tool_choice(
|
| 556 |
+
model=model, custom_llm_provider=self.custom_llm_provider
|
| 557 |
+
) or litellm.utils.supports_tool_choice(
|
| 558 |
+
model=base_model, custom_llm_provider=self.custom_llm_provider
|
| 559 |
+
):
|
| 560 |
+
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
|
| 561 |
+
supported_params.append("tool_choice")
|
| 562 |
+
|
| 563 |
+
if "gpt-oss" in model:
|
| 564 |
+
supported_params.append("reasoning_effort")
|
| 565 |
+
elif self._is_nova_2_model(model):
|
| 566 |
+
# Nova 2 models support reasoning_effort (transformed to reasoningConfig)
|
| 567 |
+
# These models use a different reasoning structure than Anthropic's thinking parameter
|
| 568 |
+
supported_params.append("reasoning_effort")
|
| 569 |
+
elif (
|
| 570 |
+
"claude-3-7" in model
|
| 571 |
+
or "claude-sonnet-4" in model
|
| 572 |
+
or "claude-opus-4" in model
|
| 573 |
+
or "deepseek.r1" in model
|
| 574 |
+
or supports_reasoning(
|
| 575 |
+
model=model,
|
| 576 |
+
custom_llm_provider=self.custom_llm_provider,
|
| 577 |
+
)
|
| 578 |
+
or supports_reasoning(
|
| 579 |
+
model=base_model, custom_llm_provider=self.custom_llm_provider
|
| 580 |
+
)
|
| 581 |
+
):
|
| 582 |
+
supported_params.append("thinking")
|
| 583 |
+
supported_params.append("reasoning_effort")
|
| 584 |
+
return supported_params
|
| 585 |
+
|
| 586 |
+
def map_tool_choice_values(
|
| 587 |
+
self, model: str, tool_choice: Union[str, dict], drop_params: bool
|
| 588 |
+
) -> Optional[ToolChoiceValuesBlock]:
|
| 589 |
+
if tool_choice == "none":
|
| 590 |
+
if litellm.drop_params is True or drop_params is True:
|
| 591 |
+
return None
|
| 592 |
+
else:
|
| 593 |
+
raise litellm.utils.UnsupportedParamsError(
|
| 594 |
+
message="Bedrock doesn't support tool_choice={}. To drop it from the call, set `litellm.drop_params = True.".format(
|
| 595 |
+
tool_choice
|
| 596 |
+
),
|
| 597 |
+
status_code=400,
|
| 598 |
+
)
|
| 599 |
+
elif tool_choice == "required":
|
| 600 |
+
return ToolChoiceValuesBlock(any={})
|
| 601 |
+
elif tool_choice == "auto":
|
| 602 |
+
return ToolChoiceValuesBlock(auto={})
|
| 603 |
+
elif isinstance(tool_choice, dict):
|
| 604 |
+
# only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
|
| 605 |
+
specific_tool = SpecificToolChoiceBlock(
|
| 606 |
+
name=tool_choice.get("function", {}).get("name", "")
|
| 607 |
+
)
|
| 608 |
+
return ToolChoiceValuesBlock(tool=specific_tool)
|
| 609 |
+
else:
|
| 610 |
+
raise litellm.utils.UnsupportedParamsError(
|
| 611 |
+
message="Bedrock doesn't support tool_choice={}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.".format(
|
| 612 |
+
tool_choice
|
| 613 |
+
),
|
| 614 |
+
status_code=400,
|
| 615 |
+
)
|
| 616 |
+
|
| 617 |
+
def get_supported_image_types(self) -> List[str]:
|
| 618 |
+
return ["png", "jpeg", "gif", "webp"]
|
| 619 |
+
|
| 620 |
+
def get_supported_document_types(self) -> List[str]:
|
| 621 |
+
return ["pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"]
|
| 622 |
+
|
| 623 |
+
def get_supported_video_types(self) -> List[str]:
|
| 624 |
+
return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"]
|
| 625 |
+
|
| 626 |
+
def get_all_supported_content_types(self) -> List[str]:
|
| 627 |
+
return (
|
| 628 |
+
self.get_supported_image_types()
|
| 629 |
+
+ self.get_supported_document_types()
|
| 630 |
+
+ self.get_supported_video_types()
|
| 631 |
+
)
|
| 632 |
+
|
| 633 |
+
def is_computer_use_tool_used(
|
| 634 |
+
self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str
|
| 635 |
+
) -> bool:
|
| 636 |
+
"""Check if computer use tools are being used in the request."""
|
| 637 |
+
if tools is None:
|
| 638 |
+
return False
|
| 639 |
+
|
| 640 |
+
for tool in tools:
|
| 641 |
+
if "type" in tool:
|
| 642 |
+
tool_type = tool["type"]
|
| 643 |
+
for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS:
|
| 644 |
+
if tool_type.startswith(computer_use_prefix):
|
| 645 |
+
return True
|
| 646 |
+
return False
|
| 647 |
+
|
| 648 |
+
def _transform_computer_use_tools(
|
| 649 |
+
self, computer_use_tools: List[OpenAIChatCompletionToolParam]
|
| 650 |
+
) -> List[dict]:
|
| 651 |
+
"""Transform computer use tools to Bedrock format."""
|
| 652 |
+
transformed_tools: List[dict] = []
|
| 653 |
+
|
| 654 |
+
for tool in computer_use_tools:
|
| 655 |
+
tool_type = tool.get("type", "")
|
| 656 |
+
|
| 657 |
+
# Check if this is a computer use tool with the startswith method
|
| 658 |
+
is_computer_use_tool = False
|
| 659 |
+
for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS:
|
| 660 |
+
if tool_type.startswith(computer_use_prefix):
|
| 661 |
+
is_computer_use_tool = True
|
| 662 |
+
break
|
| 663 |
+
|
| 664 |
+
transformed_tool: dict = {}
|
| 665 |
+
if is_computer_use_tool:
|
| 666 |
+
if tool_type.startswith("computer_") and "function" in tool:
|
| 667 |
+
# Computer use tool with function format
|
| 668 |
+
func = tool["function"]
|
| 669 |
+
transformed_tool = {
|
| 670 |
+
"type": tool_type,
|
| 671 |
+
"name": func.get("name", "computer"),
|
| 672 |
+
**func.get("parameters", {}),
|
| 673 |
+
}
|
| 674 |
+
else:
|
| 675 |
+
# Direct tools - just need to ensure name is present
|
| 676 |
+
transformed_tool = dict(tool)
|
| 677 |
+
if "name" not in transformed_tool:
|
| 678 |
+
if tool_type.startswith("bash_"):
|
| 679 |
+
transformed_tool["name"] = "bash"
|
| 680 |
+
elif tool_type.startswith("text_editor_"):
|
| 681 |
+
transformed_tool["name"] = "str_replace_editor"
|
| 682 |
+
else:
|
| 683 |
+
# Pass through other tools as-is
|
| 684 |
+
transformed_tool = dict(tool)
|
| 685 |
+
|
| 686 |
+
transformed_tools.append(transformed_tool)
|
| 687 |
+
|
| 688 |
+
return transformed_tools
|
| 689 |
+
|
| 690 |
+
def _separate_computer_use_tools(
|
| 691 |
+
self, tools: List[OpenAIChatCompletionToolParam], model: str
|
| 692 |
+
) -> Tuple[
|
| 693 |
+
List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]
|
| 694 |
+
]:
|
| 695 |
+
"""
|
| 696 |
+
Separate computer use tools from regular function tools.
|
| 697 |
+
|
| 698 |
+
Args:
|
| 699 |
+
tools: List of tools to separate
|
| 700 |
+
model: The model name to check if it supports computer use
|
| 701 |
+
|
| 702 |
+
Returns:
|
| 703 |
+
Tuple of (computer_use_tools, regular_tools)
|
| 704 |
+
"""
|
| 705 |
+
computer_use_tools = []
|
| 706 |
+
regular_tools = []
|
| 707 |
+
|
| 708 |
+
for tool in tools:
|
| 709 |
+
if "type" in tool:
|
| 710 |
+
tool_type = tool["type"]
|
| 711 |
+
is_computer_use_tool = False
|
| 712 |
+
for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS:
|
| 713 |
+
if tool_type.startswith(computer_use_prefix):
|
| 714 |
+
is_computer_use_tool = True
|
| 715 |
+
break
|
| 716 |
+
if is_computer_use_tool:
|
| 717 |
+
computer_use_tools.append(tool)
|
| 718 |
+
else:
|
| 719 |
+
regular_tools.append(tool)
|
| 720 |
+
else:
|
| 721 |
+
regular_tools.append(tool)
|
| 722 |
+
|
| 723 |
+
return computer_use_tools, regular_tools
|
| 724 |
+
|
| 725 |
+
def _create_json_tool_call_for_response_format(
|
| 726 |
+
self,
|
| 727 |
+
json_schema: Optional[dict] = None,
|
| 728 |
+
description: Optional[str] = None,
|
| 729 |
+
) -> ChatCompletionToolParam:
|
| 730 |
+
"""
|
| 731 |
+
Handles creating a tool call for getting responses in JSON format.
|
| 732 |
+
|
| 733 |
+
Args:
|
| 734 |
+
json_schema (Optional[dict]): The JSON schema the response should be in
|
| 735 |
+
|
| 736 |
+
Returns:
|
| 737 |
+
AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format
|
| 738 |
+
"""
|
| 739 |
+
|
| 740 |
+
if json_schema is None:
|
| 741 |
+
# Anthropic raises a 400 BadRequest error if properties is passed as None
|
| 742 |
+
# see usage with additionalProperties (Example 5) https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb
|
| 743 |
+
_input_schema = {
|
| 744 |
+
"type": "object",
|
| 745 |
+
"additionalProperties": True,
|
| 746 |
+
"properties": {},
|
| 747 |
+
}
|
| 748 |
+
else:
|
| 749 |
+
# Use the schema as-is for Bedrock
|
| 750 |
+
# Bedrock requires the tool schema to be of type "object" and doesn't need unwrapping
|
| 751 |
+
_input_schema = json_schema
|
| 752 |
+
|
| 753 |
+
tool_param_function_chunk = ChatCompletionToolParamFunctionChunk(
|
| 754 |
+
name=RESPONSE_FORMAT_TOOL_NAME, parameters=_input_schema
|
| 755 |
+
)
|
| 756 |
+
if description:
|
| 757 |
+
tool_param_function_chunk["description"] = description
|
| 758 |
+
|
| 759 |
+
_tool = ChatCompletionToolParam(
|
| 760 |
+
type="function",
|
| 761 |
+
function=tool_param_function_chunk,
|
| 762 |
+
)
|
| 763 |
+
return _tool
|
| 764 |
+
|
| 765 |
+
@staticmethod
|
| 766 |
+
def _supports_native_structured_outputs(model: str) -> bool:
|
| 767 |
+
"""Check if the Bedrock model supports native structured outputs (outputConfig.textFormat)."""
|
| 768 |
+
return any(
|
| 769 |
+
substring in model for substring in BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS
|
| 770 |
+
)
|
| 771 |
+
|
| 772 |
+
@staticmethod
|
| 773 |
+
def _add_additional_properties_to_schema(schema: dict) -> dict:
|
| 774 |
+
"""
|
| 775 |
+
Recursively ensure all object types in a JSON schema have
|
| 776 |
+
``"additionalProperties": false``.
|
| 777 |
+
|
| 778 |
+
Bedrock's native structured-outputs API requires this field to be
|
| 779 |
+
explicitly set on every object node, otherwise it returns a
|
| 780 |
+
validation error.
|
| 781 |
+
"""
|
| 782 |
+
if not isinstance(schema, dict):
|
| 783 |
+
return schema
|
| 784 |
+
|
| 785 |
+
result = dict(schema)
|
| 786 |
+
|
| 787 |
+
if result.get("type") == "object" and "additionalProperties" not in result:
|
| 788 |
+
result["additionalProperties"] = False
|
| 789 |
+
|
| 790 |
+
# Recurse into nested schemas
|
| 791 |
+
if "properties" in result and isinstance(result["properties"], dict):
|
| 792 |
+
result["properties"] = {
|
| 793 |
+
k: AmazonConverseConfig._add_additional_properties_to_schema(v)
|
| 794 |
+
for k, v in result["properties"].items()
|
| 795 |
+
}
|
| 796 |
+
if "items" in result and isinstance(result["items"], dict):
|
| 797 |
+
result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(
|
| 798 |
+
result["items"]
|
| 799 |
+
)
|
| 800 |
+
for defs_key in ("$defs", "definitions"):
|
| 801 |
+
if defs_key in result and isinstance(result[defs_key], dict):
|
| 802 |
+
result[defs_key] = {
|
| 803 |
+
k: AmazonConverseConfig._add_additional_properties_to_schema(v)
|
| 804 |
+
for k, v in result[defs_key].items()
|
| 805 |
+
}
|
| 806 |
+
for key in ("anyOf", "allOf", "oneOf"):
|
| 807 |
+
if key in result and isinstance(result[key], list):
|
| 808 |
+
result[key] = [
|
| 809 |
+
AmazonConverseConfig._add_additional_properties_to_schema(item)
|
| 810 |
+
for item in result[key]
|
| 811 |
+
]
|
| 812 |
+
|
| 813 |
+
return result
|
| 814 |
+
|
| 815 |
+
@staticmethod
|
| 816 |
+
def _create_output_config_for_response_format(
|
| 817 |
+
json_schema: Optional[dict] = None,
|
| 818 |
+
name: Optional[str] = None,
|
| 819 |
+
description: Optional[str] = None,
|
| 820 |
+
) -> "OutputConfigBlock":
|
| 821 |
+
"""
|
| 822 |
+
Build an outputConfig block for Bedrock's native structured outputs API.
|
| 823 |
+
|
| 824 |
+
The Converse API expects:
|
| 825 |
+
{
|
| 826 |
+
"outputConfig": {
|
| 827 |
+
"textFormat": {
|
| 828 |
+
"type": "json_schema",
|
| 829 |
+
"structure": {
|
| 830 |
+
"jsonSchema": {
|
| 831 |
+
"schema": "<json-string>",
|
| 832 |
+
"name": "optional",
|
| 833 |
+
"description": "optional"
|
| 834 |
+
}
|
| 835 |
+
}
|
| 836 |
+
}
|
| 837 |
+
}
|
| 838 |
+
}
|
| 839 |
+
"""
|
| 840 |
+
if json_schema is not None:
|
| 841 |
+
json_schema = AmazonConverseConfig._add_additional_properties_to_schema(
|
| 842 |
+
json_schema
|
| 843 |
+
)
|
| 844 |
+
schema_str = json.dumps(json_schema) if json_schema is not None else "{}"
|
| 845 |
+
json_schema_def: JsonSchemaDefinition = {"schema": schema_str}
|
| 846 |
+
if name is not None:
|
| 847 |
+
json_schema_def["name"] = name
|
| 848 |
+
if description is not None:
|
| 849 |
+
json_schema_def["description"] = description
|
| 850 |
+
|
| 851 |
+
return OutputConfigBlock(
|
| 852 |
+
textFormat=OutputFormat(
|
| 853 |
+
type="json_schema",
|
| 854 |
+
structure=OutputFormatStructure(jsonSchema=json_schema_def),
|
| 855 |
+
)
|
| 856 |
+
)
|
| 857 |
+
|
| 858 |
+
def _apply_tool_call_transformation(
|
| 859 |
+
self,
|
| 860 |
+
tools: List[OpenAIChatCompletionToolParam],
|
| 861 |
+
model: str,
|
| 862 |
+
non_default_params: dict,
|
| 863 |
+
optional_params: dict,
|
| 864 |
+
):
|
| 865 |
+
optional_params = self._add_tools_to_optional_params(
|
| 866 |
+
optional_params=optional_params, tools=tools
|
| 867 |
+
)
|
| 868 |
+
|
| 869 |
+
if (
|
| 870 |
+
"meta.llama3-3-70b-instruct-v1:0" in model
|
| 871 |
+
and non_default_params.get("stream", False) is True
|
| 872 |
+
):
|
| 873 |
+
optional_params["fake_stream"] = True
|
| 874 |
+
|
| 875 |
+
def map_openai_params(
|
| 876 |
+
self,
|
| 877 |
+
non_default_params: dict,
|
| 878 |
+
optional_params: dict,
|
| 879 |
+
model: str,
|
| 880 |
+
drop_params: bool,
|
| 881 |
+
) -> dict:
|
| 882 |
+
is_thinking_enabled = self.is_thinking_enabled(non_default_params)
|
| 883 |
+
|
| 884 |
+
for param, value in non_default_params.items():
|
| 885 |
+
if param == "response_format" and isinstance(value, dict):
|
| 886 |
+
optional_params = self._translate_response_format_param(
|
| 887 |
+
value=value,
|
| 888 |
+
model=model,
|
| 889 |
+
optional_params=optional_params,
|
| 890 |
+
non_default_params=non_default_params,
|
| 891 |
+
is_thinking_enabled=is_thinking_enabled,
|
| 892 |
+
)
|
| 893 |
+
if param == "max_tokens" or param == "max_completion_tokens":
|
| 894 |
+
optional_params["maxTokens"] = value
|
| 895 |
+
if param == "stream":
|
| 896 |
+
optional_params["stream"] = value
|
| 897 |
+
if param == "stop":
|
| 898 |
+
if isinstance(value, str):
|
| 899 |
+
if len(value) == 0: # converse raises error for empty strings
|
| 900 |
+
continue
|
| 901 |
+
value = [value]
|
| 902 |
+
optional_params["stopSequences"] = value
|
| 903 |
+
if param == "temperature":
|
| 904 |
+
optional_params["temperature"] = value
|
| 905 |
+
if param == "top_p":
|
| 906 |
+
optional_params["topP"] = value
|
| 907 |
+
if param == "tools" and isinstance(value, list):
|
| 908 |
+
self._apply_tool_call_transformation(
|
| 909 |
+
tools=cast(List[OpenAIChatCompletionToolParam], value),
|
| 910 |
+
model=model,
|
| 911 |
+
non_default_params=non_default_params,
|
| 912 |
+
optional_params=optional_params,
|
| 913 |
+
)
|
| 914 |
+
if param == "tool_choice":
|
| 915 |
+
_tool_choice_value = self.map_tool_choice_values(
|
| 916 |
+
model=model, tool_choice=value, drop_params=drop_params # type: ignore
|
| 917 |
+
)
|
| 918 |
+
if _tool_choice_value is not None:
|
| 919 |
+
optional_params["tool_choice"] = _tool_choice_value
|
| 920 |
+
if param == "parallel_tool_calls":
|
| 921 |
+
disable_parallel = not value
|
| 922 |
+
optional_params["_parallel_tool_use_config"] = {
|
| 923 |
+
"tool_choice": {"disable_parallel_tool_use": disable_parallel}
|
| 924 |
+
}
|
| 925 |
+
if param == "thinking":
|
| 926 |
+
optional_params["thinking"] = value
|
| 927 |
+
elif param == "reasoning_effort" and isinstance(value, str):
|
| 928 |
+
self._handle_reasoning_effort_parameter(
|
| 929 |
+
model=model, reasoning_effort=value, optional_params=optional_params
|
| 930 |
+
)
|
| 931 |
+
if param == "requestMetadata":
|
| 932 |
+
if value is not None and isinstance(value, dict):
|
| 933 |
+
self._validate_request_metadata(value) # type: ignore
|
| 934 |
+
optional_params["requestMetadata"] = value
|
| 935 |
+
if param == "service_tier" and isinstance(value, str):
|
| 936 |
+
self._map_service_tier_param(value, optional_params)
|
| 937 |
+
|
| 938 |
+
if param == "web_search_options" and isinstance(value, dict):
|
| 939 |
+
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
|
| 940 |
+
# because empty dict {} is falsy but is a valid way to enable Nova grounding
|
| 941 |
+
grounding_tool = self._map_web_search_options(value, model)
|
| 942 |
+
if grounding_tool is not None:
|
| 943 |
+
optional_params = self._add_tools_to_optional_params(
|
| 944 |
+
optional_params=optional_params, tools=[grounding_tool]
|
| 945 |
+
)
|
| 946 |
+
|
| 947 |
+
# Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
|
| 948 |
+
# Nova 2 handles token budgeting differently through reasoningConfig
|
| 949 |
+
if "gpt-oss" not in model and not self._is_nova_2_model(model):
|
| 950 |
+
self.update_optional_params_with_thinking_tokens(
|
| 951 |
+
non_default_params=non_default_params, optional_params=optional_params
|
| 952 |
+
)
|
| 953 |
+
|
| 954 |
+
final_is_thinking_enabled = self.is_thinking_enabled(optional_params)
|
| 955 |
+
if final_is_thinking_enabled and "tool_choice" in optional_params:
|
| 956 |
+
tool_choice_block = optional_params["tool_choice"]
|
| 957 |
+
if isinstance(tool_choice_block, dict):
|
| 958 |
+
if "any" in tool_choice_block or "tool" in tool_choice_block:
|
| 959 |
+
verbose_logger.info(
|
| 960 |
+
f"{model} does not support forced tool use (tool_choice='required' or specific tool) "
|
| 961 |
+
f"when reasoning is enabled. Changing tool_choice to 'auto'."
|
| 962 |
+
)
|
| 963 |
+
optional_params["tool_choice"] = ToolChoiceValuesBlock(auto={})
|
| 964 |
+
|
| 965 |
+
return optional_params
|
| 966 |
+
|
| 967 |
+
def _map_service_tier_param(self, value: str, optional_params: dict) -> None:
|
| 968 |
+
"""Map OpenAI service_tier (string) to Bedrock serviceTier (object).
|
| 969 |
+
|
| 970 |
+
OpenAI values: "auto", "default", "flex", "priority"
|
| 971 |
+
Bedrock values: "default", "flex", "priority" (no "auto")
|
| 972 |
+
"""
|
| 973 |
+
bedrock_tier = value
|
| 974 |
+
if value == "auto":
|
| 975 |
+
bedrock_tier = "default" # Bedrock doesn't support "auto"
|
| 976 |
+
if bedrock_tier in ("default", "flex", "priority"):
|
| 977 |
+
optional_params["serviceTier"] = {"type": bedrock_tier}
|
| 978 |
+
|
| 979 |
+
def _translate_response_format_param(
|
| 980 |
+
self,
|
| 981 |
+
value: dict,
|
| 982 |
+
model: str,
|
| 983 |
+
optional_params: dict,
|
| 984 |
+
non_default_params: dict,
|
| 985 |
+
is_thinking_enabled: bool,
|
| 986 |
+
) -> dict:
|
| 987 |
+
"""
|
| 988 |
+
Handles translation of response_format parameter to Bedrock format.
|
| 989 |
+
|
| 990 |
+
Returns `optional_params` with the translated response_format parameter.
|
| 991 |
+
"""
|
| 992 |
+
ignore_response_format_types = ["text"]
|
| 993 |
+
if value["type"] in ignore_response_format_types: # value is a no-op
|
| 994 |
+
return optional_params
|
| 995 |
+
|
| 996 |
+
json_schema: Optional[dict] = None
|
| 997 |
+
name: Optional[str] = None
|
| 998 |
+
description: Optional[str] = None
|
| 999 |
+
if "response_schema" in value:
|
| 1000 |
+
json_schema = value["response_schema"]
|
| 1001 |
+
elif "json_schema" in value:
|
| 1002 |
+
json_schema = value["json_schema"]["schema"]
|
| 1003 |
+
name = value["json_schema"].get("name")
|
| 1004 |
+
description = value["json_schema"].get("description")
|
| 1005 |
+
|
| 1006 |
+
if "type" in value and value["type"] == "text":
|
| 1007 |
+
return optional_params
|
| 1008 |
+
|
| 1009 |
+
if self._supports_native_structured_outputs(model) and json_schema is not None:
|
| 1010 |
+
# Use Bedrock's native structured outputs API (outputConfig.textFormat)
|
| 1011 |
+
# No synthetic tool injection, no fake_stream needed.
|
| 1012 |
+
# Requires an explicit schema — json_object with no schema falls through
|
| 1013 |
+
# to the tool-call path below.
|
| 1014 |
+
output_config = self._create_output_config_for_response_format(
|
| 1015 |
+
json_schema=json_schema,
|
| 1016 |
+
name=name,
|
| 1017 |
+
description=description,
|
| 1018 |
+
)
|
| 1019 |
+
optional_params["outputConfig"] = output_config
|
| 1020 |
+
else:
|
| 1021 |
+
# Fallback: translate to a synthetic tool call
|
| 1022 |
+
# https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
|
| 1023 |
+
_tool = self._create_json_tool_call_for_response_format(
|
| 1024 |
+
json_schema=json_schema,
|
| 1025 |
+
description=description,
|
| 1026 |
+
)
|
| 1027 |
+
optional_params = self._add_tools_to_optional_params(
|
| 1028 |
+
optional_params=optional_params, tools=[_tool]
|
| 1029 |
+
)
|
| 1030 |
+
|
| 1031 |
+
if (
|
| 1032 |
+
litellm.utils.supports_tool_choice(
|
| 1033 |
+
model=model, custom_llm_provider=self.custom_llm_provider
|
| 1034 |
+
)
|
| 1035 |
+
and not is_thinking_enabled
|
| 1036 |
+
):
|
| 1037 |
+
optional_params["tool_choice"] = ToolChoiceValuesBlock(
|
| 1038 |
+
tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
|
| 1039 |
+
)
|
| 1040 |
+
if non_default_params.get("stream", False) is True:
|
| 1041 |
+
optional_params["fake_stream"] = True
|
| 1042 |
+
|
| 1043 |
+
optional_params["json_mode"] = True
|
| 1044 |
+
return optional_params
|
| 1045 |
+
|
| 1046 |
+
def update_optional_params_with_thinking_tokens(
|
| 1047 |
+
self, non_default_params: dict, optional_params: dict
|
| 1048 |
+
):
|
| 1049 |
+
"""
|
| 1050 |
+
Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.
|
| 1051 |
+
|
| 1052 |
+
Checks 'non_default_params' for 'thinking' and 'max_tokens'
|
| 1053 |
+
|
| 1054 |
+
if 'thinking' is enabled and 'max_tokens' is not specified, set 'max_tokens' to the thinking token budget + DEFAULT_MAX_TOKENS
|
| 1055 |
+
|
| 1056 |
+
Also clamps thinking.budget_tokens to the Bedrock minimum (1024) to
|
| 1057 |
+
prevent 400 errors from the Bedrock API.
|
| 1058 |
+
"""
|
| 1059 |
+
from litellm.constants import DEFAULT_MAX_TOKENS
|
| 1060 |
+
|
| 1061 |
+
self._clamp_thinking_budget_tokens(optional_params)
|
| 1062 |
+
|
| 1063 |
+
is_thinking_enabled = self.is_thinking_enabled(optional_params)
|
| 1064 |
+
is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params)
|
| 1065 |
+
if is_thinking_enabled and not is_max_tokens_in_request:
|
| 1066 |
+
thinking_token_budget = cast(dict, optional_params["thinking"]).get(
|
| 1067 |
+
"budget_tokens", None
|
| 1068 |
+
)
|
| 1069 |
+
if thinking_token_budget is not None:
|
| 1070 |
+
optional_params["maxTokens"] = (
|
| 1071 |
+
thinking_token_budget + DEFAULT_MAX_TOKENS
|
| 1072 |
+
)
|
| 1073 |
+
|
| 1074 |
+
@overload
|
| 1075 |
+
def _get_cache_point_block(
|
| 1076 |
+
self,
|
| 1077 |
+
message_block: Union[
|
| 1078 |
+
OpenAIMessageContentListBlock,
|
| 1079 |
+
ChatCompletionUserMessage,
|
| 1080 |
+
ChatCompletionSystemMessage,
|
| 1081 |
+
ChatCompletionAssistantMessage,
|
| 1082 |
+
],
|
| 1083 |
+
block_type: Literal["system"],
|
| 1084 |
+
model: Optional[str] = None,
|
| 1085 |
+
) -> Optional[SystemContentBlock]:
|
| 1086 |
+
pass
|
| 1087 |
+
|
| 1088 |
+
@overload
|
| 1089 |
+
def _get_cache_point_block(
|
| 1090 |
+
self,
|
| 1091 |
+
message_block: Union[
|
| 1092 |
+
OpenAIMessageContentListBlock,
|
| 1093 |
+
ChatCompletionUserMessage,
|
| 1094 |
+
ChatCompletionSystemMessage,
|
| 1095 |
+
ChatCompletionAssistantMessage,
|
| 1096 |
+
],
|
| 1097 |
+
block_type: Literal["content_block"],
|
| 1098 |
+
model: Optional[str] = None,
|
| 1099 |
+
) -> Optional[ContentBlock]:
|
| 1100 |
+
pass
|
| 1101 |
+
|
| 1102 |
+
def _get_cache_point_block(
|
| 1103 |
+
self,
|
| 1104 |
+
message_block: Union[
|
| 1105 |
+
OpenAIMessageContentListBlock,
|
| 1106 |
+
ChatCompletionUserMessage,
|
| 1107 |
+
ChatCompletionSystemMessage,
|
| 1108 |
+
ChatCompletionAssistantMessage,
|
| 1109 |
+
],
|
| 1110 |
+
block_type: Literal["system", "content_block"],
|
| 1111 |
+
model: Optional[str] = None,
|
| 1112 |
+
) -> Optional[Union[SystemContentBlock, ContentBlock]]:
|
| 1113 |
+
cache_control = message_block.get("cache_control", None)
|
| 1114 |
+
if cache_control is None:
|
| 1115 |
+
return None
|
| 1116 |
+
|
| 1117 |
+
cache_point = CachePointBlock(type="default")
|
| 1118 |
+
if isinstance(cache_control, dict) and "ttl" in cache_control:
|
| 1119 |
+
ttl = cache_control["ttl"]
|
| 1120 |
+
if ttl in ["5m", "1h"] and model is not None:
|
| 1121 |
+
if is_claude_4_5_on_bedrock(model):
|
| 1122 |
+
cache_point["ttl"] = ttl
|
| 1123 |
+
|
| 1124 |
+
if block_type == "system":
|
| 1125 |
+
return SystemContentBlock(cachePoint=cache_point)
|
| 1126 |
+
else:
|
| 1127 |
+
return ContentBlock(cachePoint=cache_point)
|
| 1128 |
+
|
| 1129 |
+
def _transform_system_message(
|
| 1130 |
+
self, messages: List[AllMessageValues], model: Optional[str] = None
|
| 1131 |
+
) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
|
| 1132 |
+
system_prompt_indices = []
|
| 1133 |
+
system_content_blocks: List[SystemContentBlock] = []
|
| 1134 |
+
for idx, message in enumerate(messages):
|
| 1135 |
+
if message["role"] == "system":
|
| 1136 |
+
system_prompt_indices.append(idx)
|
| 1137 |
+
if isinstance(message["content"], str) and message["content"]:
|
| 1138 |
+
system_content_blocks.append(
|
| 1139 |
+
SystemContentBlock(text=message["content"])
|
| 1140 |
+
)
|
| 1141 |
+
cache_block = self._get_cache_point_block(
|
| 1142 |
+
message, block_type="system", model=model
|
| 1143 |
+
)
|
| 1144 |
+
if cache_block:
|
| 1145 |
+
system_content_blocks.append(cache_block)
|
| 1146 |
+
elif isinstance(message["content"], list):
|
| 1147 |
+
for m in message["content"]:
|
| 1148 |
+
if m.get("type") == "text" and m.get("text"):
|
| 1149 |
+
system_content_blocks.append(
|
| 1150 |
+
SystemContentBlock(text=m["text"])
|
| 1151 |
+
)
|
| 1152 |
+
cache_block = self._get_cache_point_block(
|
| 1153 |
+
m, block_type="system", model=model
|
| 1154 |
+
)
|
| 1155 |
+
if cache_block:
|
| 1156 |
+
system_content_blocks.append(cache_block)
|
| 1157 |
+
if len(system_prompt_indices) > 0:
|
| 1158 |
+
for idx in reversed(system_prompt_indices):
|
| 1159 |
+
messages.pop(idx)
|
| 1160 |
+
return messages, system_content_blocks
|
| 1161 |
+
|
| 1162 |
+
def _transform_inference_params(self, inference_params: dict) -> InferenceConfig:
|
| 1163 |
+
if "top_k" in inference_params:
|
| 1164 |
+
inference_params["topK"] = inference_params.pop("top_k")
|
| 1165 |
+
return InferenceConfig(**inference_params)
|
| 1166 |
+
|
| 1167 |
+
def _handle_top_k_value(self, model: str, inference_params: dict) -> dict:
|
| 1168 |
+
base_model = BedrockModelInfo.get_base_model(model)
|
| 1169 |
+
|
| 1170 |
+
val_top_k = None
|
| 1171 |
+
if "topK" in inference_params:
|
| 1172 |
+
val_top_k = inference_params.pop("topK")
|
| 1173 |
+
elif "top_k" in inference_params:
|
| 1174 |
+
val_top_k = inference_params.pop("top_k")
|
| 1175 |
+
|
| 1176 |
+
if val_top_k:
|
| 1177 |
+
if base_model.startswith("anthropic"):
|
| 1178 |
+
return {"top_k": val_top_k}
|
| 1179 |
+
if base_model.startswith("amazon.nova"):
|
| 1180 |
+
return {"inferenceConfig": {"topK": val_top_k}}
|
| 1181 |
+
|
| 1182 |
+
return {}
|
| 1183 |
+
|
| 1184 |
+
def _prepare_request_params(
|
| 1185 |
+
self, optional_params: dict, model: str
|
| 1186 |
+
) -> Tuple[dict, dict, dict, Optional[OutputConfigBlock]]:
|
| 1187 |
+
"""Prepare and separate request parameters."""
|
| 1188 |
+
# Filter out exception objects before deepcopy to prevent deepcopy failures
|
| 1189 |
+
# Exceptions should not be stored in optional_params (this is a defensive fix)
|
| 1190 |
+
cleaned_params = filter_exceptions_from_params(optional_params)
|
| 1191 |
+
inference_params = safe_deep_copy(cleaned_params)
|
| 1192 |
+
supported_converse_params = list(
|
| 1193 |
+
AmazonConverseConfig.__annotations__.keys()
|
| 1194 |
+
) + ["top_k"]
|
| 1195 |
+
supported_tool_call_params = ["tools", "tool_choice"]
|
| 1196 |
+
supported_config_params = list(self.get_config_blocks().keys())
|
| 1197 |
+
total_supported_params = (
|
| 1198 |
+
supported_converse_params
|
| 1199 |
+
+ supported_tool_call_params
|
| 1200 |
+
+ supported_config_params
|
| 1201 |
+
)
|
| 1202 |
+
inference_params.pop("json_mode", None) # used for handling json_schema
|
| 1203 |
+
# Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and
|
| 1204 |
+
# will reject `output_config` if it leaks through pass-through routes.
|
| 1205 |
+
inference_params.pop("output_config", None)
|
| 1206 |
+
|
| 1207 |
+
# Extract requestMetadata before processing other parameters
|
| 1208 |
+
request_metadata = inference_params.pop("requestMetadata", None)
|
| 1209 |
+
if request_metadata is not None:
|
| 1210 |
+
self._validate_request_metadata(request_metadata)
|
| 1211 |
+
|
| 1212 |
+
output_config: Optional[OutputConfigBlock] = inference_params.pop(
|
| 1213 |
+
"outputConfig", None
|
| 1214 |
+
)
|
| 1215 |
+
inference_params.pop(
|
| 1216 |
+
"output_config", None
|
| 1217 |
+
) # Bedrock Converse doesn't support it
|
| 1218 |
+
|
| 1219 |
+
# keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params'
|
| 1220 |
+
additional_request_params = {
|
| 1221 |
+
k: v for k, v in inference_params.items() if k not in total_supported_params
|
| 1222 |
+
}
|
| 1223 |
+
inference_params = {
|
| 1224 |
+
k: v for k, v in inference_params.items() if k in total_supported_params
|
| 1225 |
+
}
|
| 1226 |
+
|
| 1227 |
+
# Handle parallel_tool_calls configuration
|
| 1228 |
+
parallel_tool_use_config = additional_request_params.pop(
|
| 1229 |
+
"_parallel_tool_use_config", None
|
| 1230 |
+
)
|
| 1231 |
+
if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
|
| 1232 |
+
for key, value in parallel_tool_use_config.items():
|
| 1233 |
+
if (
|
| 1234 |
+
key in additional_request_params
|
| 1235 |
+
and isinstance(additional_request_params[key], dict)
|
| 1236 |
+
and isinstance(value, dict)
|
| 1237 |
+
):
|
| 1238 |
+
additional_request_params[key].update(value)
|
| 1239 |
+
else:
|
| 1240 |
+
additional_request_params[key] = value
|
| 1241 |
+
|
| 1242 |
+
additional_request_params.pop("parallel_tool_calls", None)
|
| 1243 |
+
|
| 1244 |
+
# Only set the topK value in for models that support it
|
| 1245 |
+
additional_request_params.update(
|
| 1246 |
+
self._handle_top_k_value(model, inference_params)
|
| 1247 |
+
)
|
| 1248 |
+
|
| 1249 |
+
# Filter out internal/MCP-related parameters that shouldn't be sent to the API
|
| 1250 |
+
# These are LiteLLM internal parameters, not API parameters
|
| 1251 |
+
additional_request_params = filter_internal_params(additional_request_params)
|
| 1252 |
+
|
| 1253 |
+
# Filter out non-serializable objects (exceptions, callables, logging objects, etc.)
|
| 1254 |
+
# from additional_request_params to prevent JSON serialization errors
|
| 1255 |
+
# This filters: Exception objects, callable objects (functions), Logging objects, etc.
|
| 1256 |
+
additional_request_params = filter_exceptions_from_params(
|
| 1257 |
+
additional_request_params
|
| 1258 |
+
)
|
| 1259 |
+
|
| 1260 |
+
return (
|
| 1261 |
+
inference_params,
|
| 1262 |
+
additional_request_params,
|
| 1263 |
+
request_metadata,
|
| 1264 |
+
output_config,
|
| 1265 |
+
)
|
| 1266 |
+
|
| 1267 |
+
def _process_tools_and_beta(
|
| 1268 |
+
self,
|
| 1269 |
+
original_tools: list,
|
| 1270 |
+
model: str,
|
| 1271 |
+
headers: Optional[dict],
|
| 1272 |
+
additional_request_params: dict,
|
| 1273 |
+
) -> Tuple[List[ToolBlock], list]:
|
| 1274 |
+
"""Process tools and collect anthropic_beta values."""
|
| 1275 |
+
bedrock_tools: List[ToolBlock] = []
|
| 1276 |
+
|
| 1277 |
+
# Collect anthropic_beta values from user headers
|
| 1278 |
+
anthropic_beta_list = []
|
| 1279 |
+
if headers:
|
| 1280 |
+
user_betas = get_anthropic_beta_from_headers(headers)
|
| 1281 |
+
anthropic_beta_list.extend(user_betas)
|
| 1282 |
+
|
| 1283 |
+
# Separate pre-formatted Bedrock tools (e.g. systemTool from web_search_options)
|
| 1284 |
+
# from OpenAI-format tools that need transformation via _bedrock_tools_pt
|
| 1285 |
+
filtered_tools = []
|
| 1286 |
+
pre_formatted_tools: List[ToolBlock] = []
|
| 1287 |
+
if original_tools:
|
| 1288 |
+
for tool in original_tools:
|
| 1289 |
+
# Already-formatted Bedrock tools (e.g. systemTool for Nova grounding)
|
| 1290 |
+
if "systemTool" in tool:
|
| 1291 |
+
pre_formatted_tools.append(tool)
|
| 1292 |
+
continue
|
| 1293 |
+
tool_type = tool.get("type", "")
|
| 1294 |
+
if tool_type in (
|
| 1295 |
+
"tool_search_tool_regex_20251119",
|
| 1296 |
+
"tool_search_tool_bm25_20251119",
|
| 1297 |
+
):
|
| 1298 |
+
# Tool search not supported in Converse API - skip it
|
| 1299 |
+
continue
|
| 1300 |
+
filtered_tools.append(tool)
|
| 1301 |
+
|
| 1302 |
+
# Only separate tools if computer use tools are actually present
|
| 1303 |
+
if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model):
|
| 1304 |
+
# Separate computer use tools from regular function tools
|
| 1305 |
+
computer_use_tools, regular_tools = self._separate_computer_use_tools(
|
| 1306 |
+
filtered_tools, model
|
| 1307 |
+
)
|
| 1308 |
+
|
| 1309 |
+
# Process regular function tools using existing logic
|
| 1310 |
+
bedrock_tools = _bedrock_tools_pt(regular_tools)
|
| 1311 |
+
|
| 1312 |
+
# Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
|
| 1313 |
+
if computer_use_tools:
|
| 1314 |
+
# Determine the correct computer-use beta header based on model
|
| 1315 |
+
# "computer-use-2025-11-24" for Claude Opus 4.6, Claude Opus 4.5
|
| 1316 |
+
# "computer-use-2025-01-24" for Claude Sonnet 4.5, Haiku 4.5, Opus 4.1, Sonnet 4, Opus 4, and Sonnet 3.7
|
| 1317 |
+
# "computer-use-2024-10-22" for older models
|
| 1318 |
+
model_lower = model.lower()
|
| 1319 |
+
if (
|
| 1320 |
+
"opus-4.6" in model_lower
|
| 1321 |
+
or "opus_4.6" in model_lower
|
| 1322 |
+
or "opus-4-6" in model_lower
|
| 1323 |
+
or "opus_4_6" in model_lower
|
| 1324 |
+
or "sonnet-4.6" in model_lower
|
| 1325 |
+
or "sonnet_4.6" in model_lower
|
| 1326 |
+
or "sonnet-4-6" in model_lower
|
| 1327 |
+
or "sonnet_4_6" in model_lower
|
| 1328 |
+
):
|
| 1329 |
+
computer_use_header = "computer-use-2025-11-24"
|
| 1330 |
+
elif (
|
| 1331 |
+
"opus-4.5" in model_lower
|
| 1332 |
+
or "opus_4.5" in model_lower
|
| 1333 |
+
or "opus-4-5" in model_lower
|
| 1334 |
+
or "opus_4_5" in model_lower
|
| 1335 |
+
):
|
| 1336 |
+
computer_use_header = "computer-use-2025-11-24"
|
| 1337 |
+
elif any(
|
| 1338 |
+
pattern in model_lower
|
| 1339 |
+
for pattern in [
|
| 1340 |
+
"sonnet-4.5",
|
| 1341 |
+
"sonnet_4.5",
|
| 1342 |
+
"sonnet-4-5",
|
| 1343 |
+
"sonnet_4_5",
|
| 1344 |
+
"haiku-4.5",
|
| 1345 |
+
"haiku_4.5",
|
| 1346 |
+
"haiku-4-5",
|
| 1347 |
+
"haiku_4_5",
|
| 1348 |
+
"opus-4.1",
|
| 1349 |
+
"opus_4.1",
|
| 1350 |
+
"opus-4-1",
|
| 1351 |
+
"opus_4_1",
|
| 1352 |
+
"sonnet-4",
|
| 1353 |
+
"sonnet_4",
|
| 1354 |
+
"opus-4",
|
| 1355 |
+
"opus_4",
|
| 1356 |
+
"sonnet-3.7",
|
| 1357 |
+
"sonnet_3.7",
|
| 1358 |
+
"sonnet-3-7",
|
| 1359 |
+
"sonnet_3_7",
|
| 1360 |
+
]
|
| 1361 |
+
):
|
| 1362 |
+
computer_use_header = "computer-use-2025-01-24"
|
| 1363 |
+
else:
|
| 1364 |
+
computer_use_header = "computer-use-2024-10-22"
|
| 1365 |
+
|
| 1366 |
+
anthropic_beta_list.append(computer_use_header)
|
| 1367 |
+
# Transform computer use tools to proper Bedrock format
|
| 1368 |
+
transformed_computer_tools = self._transform_computer_use_tools(
|
| 1369 |
+
computer_use_tools
|
| 1370 |
+
)
|
| 1371 |
+
additional_request_params["tools"] = transformed_computer_tools
|
| 1372 |
+
else:
|
| 1373 |
+
# No computer use tools, process all tools as regular tools
|
| 1374 |
+
bedrock_tools = _bedrock_tools_pt(filtered_tools)
|
| 1375 |
+
|
| 1376 |
+
# Append pre-formatted tools (systemTool etc.) after transformation
|
| 1377 |
+
bedrock_tools.extend(pre_formatted_tools)
|
| 1378 |
+
|
| 1379 |
+
# Set anthropic_beta in additional_request_params if we have any beta features
|
| 1380 |
+
# ONLY apply to Anthropic/Claude models - other models (e.g., Qwen, Llama) don't support this field
|
| 1381 |
+
base_model = BedrockModelInfo.get_base_model(model)
|
| 1382 |
+
if anthropic_beta_list and base_model.startswith("anthropic"):
|
| 1383 |
+
additional_request_params["anthropic_beta"] = anthropic_beta_list
|
| 1384 |
+
|
| 1385 |
+
return bedrock_tools, anthropic_beta_list
|
| 1386 |
+
|
| 1387 |
+
def _transform_request_helper(
|
| 1388 |
+
self,
|
| 1389 |
+
model: str,
|
| 1390 |
+
system_content_blocks: List[SystemContentBlock],
|
| 1391 |
+
optional_params: dict,
|
| 1392 |
+
messages: Optional[List[AllMessageValues]] = None,
|
| 1393 |
+
headers: Optional[dict] = None,
|
| 1394 |
+
) -> CommonRequestObject:
|
| 1395 |
+
## VALIDATE REQUEST
|
| 1396 |
+
"""
|
| 1397 |
+
Bedrock doesn't support tool calling without `tools=` param specified.
|
| 1398 |
+
"""
|
| 1399 |
+
if (
|
| 1400 |
+
"tools" not in optional_params
|
| 1401 |
+
and messages is not None
|
| 1402 |
+
and has_tool_call_blocks(messages)
|
| 1403 |
+
):
|
| 1404 |
+
if litellm.modify_params:
|
| 1405 |
+
optional_params["tools"] = add_dummy_tool(
|
| 1406 |
+
custom_llm_provider="bedrock_converse"
|
| 1407 |
+
)
|
| 1408 |
+
else:
|
| 1409 |
+
raise litellm.UnsupportedParamsError(
|
| 1410 |
+
message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
|
| 1411 |
+
model="",
|
| 1412 |
+
llm_provider="bedrock",
|
| 1413 |
+
)
|
| 1414 |
+
|
| 1415 |
+
# Drop thinking param if thinking is enabled but thinking_blocks are missing
|
| 1416 |
+
# This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
|
| 1417 |
+
#
|
| 1418 |
+
# IMPORTANT: Only drop thinking if NO assistant messages have thinking_blocks.
|
| 1419 |
+
# If any message has thinking_blocks, we must keep thinking enabled, otherwise
|
| 1420 |
+
# Related issues: https://github.com/BerriAI/litellm/issues/14194
|
| 1421 |
+
if (
|
| 1422 |
+
optional_params.get("thinking") is not None
|
| 1423 |
+
and messages is not None
|
| 1424 |
+
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
|
| 1425 |
+
and not any_assistant_message_has_thinking_blocks(messages)
|
| 1426 |
+
):
|
| 1427 |
+
if litellm.modify_params:
|
| 1428 |
+
optional_params.pop("thinking", None)
|
| 1429 |
+
litellm.verbose_logger.warning(
|
| 1430 |
+
"Dropping 'thinking' param because the last assistant message with tool_calls "
|
| 1431 |
+
"has no thinking_blocks. The model won't use extended thinking for this turn."
|
| 1432 |
+
)
|
| 1433 |
+
|
| 1434 |
+
# Prepare and separate parameters
|
| 1435 |
+
(
|
| 1436 |
+
inference_params,
|
| 1437 |
+
additional_request_params,
|
| 1438 |
+
request_metadata,
|
| 1439 |
+
output_config,
|
| 1440 |
+
) = self._prepare_request_params(optional_params, model)
|
| 1441 |
+
|
| 1442 |
+
original_tools = inference_params.pop("tools", [])
|
| 1443 |
+
|
| 1444 |
+
# Process tools and collect beta values
|
| 1445 |
+
bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
|
| 1446 |
+
original_tools, model, headers, additional_request_params
|
| 1447 |
+
)
|
| 1448 |
+
|
| 1449 |
+
# Append cachePoint to tools if cache_control_injection_points has tool_config
|
| 1450 |
+
cache_injection_points = additional_request_params.pop(
|
| 1451 |
+
"cache_control_injection_points", None
|
| 1452 |
+
)
|
| 1453 |
+
if cache_injection_points and len(bedrock_tools) > 0:
|
| 1454 |
+
for point in cache_injection_points:
|
| 1455 |
+
if point.get("location") == "tool_config":
|
| 1456 |
+
bedrock_tools.append({"cachePoint": {"type": "default"}})
|
| 1457 |
+
break
|
| 1458 |
+
|
| 1459 |
+
bedrock_tool_config: Optional[ToolConfigBlock] = None
|
| 1460 |
+
if len(bedrock_tools) > 0:
|
| 1461 |
+
tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
|
| 1462 |
+
"tool_choice", None
|
| 1463 |
+
)
|
| 1464 |
+
bedrock_tool_config = ToolConfigBlock(
|
| 1465 |
+
tools=bedrock_tools,
|
| 1466 |
+
)
|
| 1467 |
+
if tool_choice_values is not None:
|
| 1468 |
+
bedrock_tool_config["toolChoice"] = tool_choice_values
|
| 1469 |
+
|
| 1470 |
+
data: CommonRequestObject = {
|
| 1471 |
+
"additionalModelRequestFields": additional_request_params,
|
| 1472 |
+
"system": system_content_blocks,
|
| 1473 |
+
"inferenceConfig": self._transform_inference_params(
|
| 1474 |
+
inference_params=inference_params
|
| 1475 |
+
),
|
| 1476 |
+
}
|
| 1477 |
+
|
| 1478 |
+
# Handle all config blocks
|
| 1479 |
+
for config_name, config_class in self.get_config_blocks().items():
|
| 1480 |
+
config_value = inference_params.pop(config_name, None)
|
| 1481 |
+
if config_value is not None:
|
| 1482 |
+
data[config_name] = config_class(**config_value) # type: ignore
|
| 1483 |
+
|
| 1484 |
+
# Tool Config
|
| 1485 |
+
if bedrock_tool_config is not None:
|
| 1486 |
+
data["toolConfig"] = bedrock_tool_config
|
| 1487 |
+
|
| 1488 |
+
# Request Metadata (top-level field)
|
| 1489 |
+
if request_metadata is not None:
|
| 1490 |
+
data["requestMetadata"] = request_metadata
|
| 1491 |
+
|
| 1492 |
+
if output_config is not None:
|
| 1493 |
+
data["outputConfig"] = output_config
|
| 1494 |
+
|
| 1495 |
+
return data
|
| 1496 |
+
|
| 1497 |
+
async def _async_transform_request(
|
| 1498 |
+
self,
|
| 1499 |
+
model: str,
|
| 1500 |
+
messages: List[AllMessageValues],
|
| 1501 |
+
optional_params: dict,
|
| 1502 |
+
litellm_params: dict,
|
| 1503 |
+
headers: Optional[dict] = None,
|
| 1504 |
+
) -> RequestObject:
|
| 1505 |
+
messages, system_content_blocks = self._transform_system_message(
|
| 1506 |
+
messages, model=model
|
| 1507 |
+
)
|
| 1508 |
+
|
| 1509 |
+
# Convert last user message to guarded_text if guardrailConfig is present
|
| 1510 |
+
messages = self._convert_consecutive_user_messages_to_guarded_text(
|
| 1511 |
+
messages, optional_params
|
| 1512 |
+
)
|
| 1513 |
+
## TRANSFORMATION ##
|
| 1514 |
+
|
| 1515 |
+
_data: CommonRequestObject = self._transform_request_helper(
|
| 1516 |
+
model=model,
|
| 1517 |
+
system_content_blocks=system_content_blocks,
|
| 1518 |
+
optional_params=optional_params,
|
| 1519 |
+
messages=messages,
|
| 1520 |
+
headers=headers,
|
| 1521 |
+
)
|
| 1522 |
+
|
| 1523 |
+
bedrock_messages = (
|
| 1524 |
+
await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
|
| 1525 |
+
messages=messages,
|
| 1526 |
+
model=model,
|
| 1527 |
+
llm_provider="bedrock_converse",
|
| 1528 |
+
user_continue_message=litellm_params.pop("user_continue_message", None),
|
| 1529 |
+
)
|
| 1530 |
+
)
|
| 1531 |
+
|
| 1532 |
+
data: RequestObject = {"messages": bedrock_messages, **_data}
|
| 1533 |
+
|
| 1534 |
+
return data
|
| 1535 |
+
|
| 1536 |
+
def transform_request(
|
| 1537 |
+
self,
|
| 1538 |
+
model: str,
|
| 1539 |
+
messages: List[AllMessageValues],
|
| 1540 |
+
optional_params: dict,
|
| 1541 |
+
litellm_params: dict,
|
| 1542 |
+
headers: dict,
|
| 1543 |
+
) -> dict:
|
| 1544 |
+
return cast(
|
| 1545 |
+
dict,
|
| 1546 |
+
self._transform_request(
|
| 1547 |
+
model=model,
|
| 1548 |
+
messages=messages,
|
| 1549 |
+
optional_params=optional_params,
|
| 1550 |
+
litellm_params=litellm_params,
|
| 1551 |
+
headers=headers,
|
| 1552 |
+
),
|
| 1553 |
+
)
|
| 1554 |
+
|
| 1555 |
+
def _transform_request(
|
| 1556 |
+
self,
|
| 1557 |
+
model: str,
|
| 1558 |
+
messages: List[AllMessageValues],
|
| 1559 |
+
optional_params: dict,
|
| 1560 |
+
litellm_params: dict,
|
| 1561 |
+
headers: Optional[dict] = None,
|
| 1562 |
+
) -> RequestObject:
|
| 1563 |
+
messages, system_content_blocks = self._transform_system_message(
|
| 1564 |
+
messages, model=model
|
| 1565 |
+
)
|
| 1566 |
+
|
| 1567 |
+
# Convert last user message to guarded_text if guardrailConfig is present
|
| 1568 |
+
messages = self._convert_consecutive_user_messages_to_guarded_text(
|
| 1569 |
+
messages, optional_params
|
| 1570 |
+
)
|
| 1571 |
+
|
| 1572 |
+
_data: CommonRequestObject = self._transform_request_helper(
|
| 1573 |
+
model=model,
|
| 1574 |
+
system_content_blocks=system_content_blocks,
|
| 1575 |
+
optional_params=optional_params,
|
| 1576 |
+
messages=messages,
|
| 1577 |
+
headers=headers,
|
| 1578 |
+
)
|
| 1579 |
+
|
| 1580 |
+
## TRANSFORMATION ##
|
| 1581 |
+
bedrock_messages: List[MessageBlock] = _bedrock_converse_messages_pt(
|
| 1582 |
+
messages=messages,
|
| 1583 |
+
model=model,
|
| 1584 |
+
llm_provider="bedrock_converse",
|
| 1585 |
+
user_continue_message=litellm_params.pop("user_continue_message", None),
|
| 1586 |
+
)
|
| 1587 |
+
|
| 1588 |
+
data: RequestObject = {"messages": bedrock_messages, **_data}
|
| 1589 |
+
|
| 1590 |
+
return data
|
| 1591 |
+
|
| 1592 |
+
def transform_response(
|
| 1593 |
+
self,
|
| 1594 |
+
model: str,
|
| 1595 |
+
raw_response: httpx.Response,
|
| 1596 |
+
model_response: ModelResponse,
|
| 1597 |
+
logging_obj: Logging,
|
| 1598 |
+
request_data: dict,
|
| 1599 |
+
messages: List[AllMessageValues],
|
| 1600 |
+
optional_params: dict,
|
| 1601 |
+
litellm_params: dict,
|
| 1602 |
+
encoding: Any,
|
| 1603 |
+
api_key: Optional[str] = None,
|
| 1604 |
+
json_mode: Optional[bool] = None,
|
| 1605 |
+
) -> ModelResponse:
|
| 1606 |
+
return self._transform_response(
|
| 1607 |
+
model=model,
|
| 1608 |
+
response=raw_response,
|
| 1609 |
+
model_response=model_response,
|
| 1610 |
+
stream=optional_params.get("stream", False),
|
| 1611 |
+
logging_obj=logging_obj,
|
| 1612 |
+
optional_params=optional_params,
|
| 1613 |
+
api_key=api_key,
|
| 1614 |
+
data=request_data,
|
| 1615 |
+
messages=messages,
|
| 1616 |
+
encoding=encoding,
|
| 1617 |
+
)
|
| 1618 |
+
|
| 1619 |
+
def _transform_reasoning_content(
|
| 1620 |
+
self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]
|
| 1621 |
+
) -> str:
|
| 1622 |
+
"""
|
| 1623 |
+
Extract the reasoning text from the reasoning content blocks
|
| 1624 |
+
|
| 1625 |
+
Ensures deepseek reasoning content compatible output.
|
| 1626 |
+
"""
|
| 1627 |
+
reasoning_content_str = ""
|
| 1628 |
+
for block in reasoning_content_blocks:
|
| 1629 |
+
if "reasoningText" in block:
|
| 1630 |
+
reasoning_content_str += block["reasoningText"]["text"]
|
| 1631 |
+
return reasoning_content_str
|
| 1632 |
+
|
| 1633 |
+
def _transform_thinking_blocks(
|
| 1634 |
+
self, thinking_blocks: List[BedrockConverseReasoningContentBlock]
|
| 1635 |
+
) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]:
|
| 1636 |
+
"""Return a consistent format for thinking blocks between Anthropic and Bedrock."""
|
| 1637 |
+
thinking_blocks_list: List[
|
| 1638 |
+
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
| 1639 |
+
] = []
|
| 1640 |
+
for block in thinking_blocks:
|
| 1641 |
+
if "reasoningText" in block:
|
| 1642 |
+
_thinking_block = ChatCompletionThinkingBlock(type="thinking")
|
| 1643 |
+
_text = block["reasoningText"].get("text")
|
| 1644 |
+
_signature = block["reasoningText"].get("signature")
|
| 1645 |
+
if _text is not None:
|
| 1646 |
+
_thinking_block["thinking"] = _text
|
| 1647 |
+
if _signature is not None:
|
| 1648 |
+
_thinking_block["signature"] = _signature
|
| 1649 |
+
thinking_blocks_list.append(_thinking_block)
|
| 1650 |
+
elif "redactedContent" in block:
|
| 1651 |
+
_redacted_block = ChatCompletionRedactedThinkingBlock(
|
| 1652 |
+
type="redacted_thinking", data=block["redactedContent"]
|
| 1653 |
+
)
|
| 1654 |
+
thinking_blocks_list.append(_redacted_block)
|
| 1655 |
+
return thinking_blocks_list
|
| 1656 |
+
|
| 1657 |
+
def _transform_usage(
|
| 1658 |
+
self,
|
| 1659 |
+
usage: ConverseTokenUsageBlock,
|
| 1660 |
+
reasoning_content: Optional[str] = None,
|
| 1661 |
+
) -> Usage:
|
| 1662 |
+
input_tokens = usage["inputTokens"]
|
| 1663 |
+
output_tokens = usage["outputTokens"]
|
| 1664 |
+
total_tokens = usage["totalTokens"]
|
| 1665 |
+
cache_creation_input_tokens: int = 0
|
| 1666 |
+
cache_read_input_tokens: int = 0
|
| 1667 |
+
|
| 1668 |
+
if "cacheReadInputTokens" in usage:
|
| 1669 |
+
cache_read_input_tokens = usage["cacheReadInputTokens"]
|
| 1670 |
+
input_tokens += cache_read_input_tokens
|
| 1671 |
+
if "cacheWriteInputTokens" in usage:
|
| 1672 |
+
cache_creation_input_tokens = usage["cacheWriteInputTokens"]
|
| 1673 |
+
input_tokens += cache_creation_input_tokens
|
| 1674 |
+
|
| 1675 |
+
prompt_tokens_details = PromptTokensDetailsWrapper(
|
| 1676 |
+
cached_tokens=cache_read_input_tokens
|
| 1677 |
+
)
|
| 1678 |
+
reasoning_tokens = (
|
| 1679 |
+
token_counter(text=reasoning_content, count_response_tokens=True)
|
| 1680 |
+
if reasoning_content
|
| 1681 |
+
else 0
|
| 1682 |
+
)
|
| 1683 |
+
completion_tokens_details = CompletionTokensDetailsWrapper(
|
| 1684 |
+
reasoning_tokens=reasoning_tokens,
|
| 1685 |
+
text_tokens=(
|
| 1686 |
+
output_tokens - reasoning_tokens
|
| 1687 |
+
if reasoning_tokens > 0
|
| 1688 |
+
else output_tokens
|
| 1689 |
+
),
|
| 1690 |
+
)
|
| 1691 |
+
openai_usage = Usage(
|
| 1692 |
+
prompt_tokens=input_tokens,
|
| 1693 |
+
completion_tokens=output_tokens,
|
| 1694 |
+
total_tokens=total_tokens,
|
| 1695 |
+
prompt_tokens_details=prompt_tokens_details,
|
| 1696 |
+
cache_creation_input_tokens=cache_creation_input_tokens,
|
| 1697 |
+
cache_read_input_tokens=cache_read_input_tokens,
|
| 1698 |
+
completion_tokens_details=completion_tokens_details,
|
| 1699 |
+
)
|
| 1700 |
+
return openai_usage
|
| 1701 |
+
|
| 1702 |
+
def get_tool_call_names(
|
| 1703 |
+
self,
|
| 1704 |
+
tools: Optional[
|
| 1705 |
+
Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]
|
| 1706 |
+
] = None,
|
| 1707 |
+
) -> List[str]:
|
| 1708 |
+
if tools is None:
|
| 1709 |
+
return []
|
| 1710 |
+
tool_set: set[str] = set()
|
| 1711 |
+
for tool in tools:
|
| 1712 |
+
tool_spec = tool.get("toolSpec")
|
| 1713 |
+
function = tool.get("function")
|
| 1714 |
+
if tool_spec is not None:
|
| 1715 |
+
_name = cast(dict, tool_spec).get("name")
|
| 1716 |
+
if _name is not None and isinstance(_name, str):
|
| 1717 |
+
tool_set.add(_name)
|
| 1718 |
+
if function is not None:
|
| 1719 |
+
_name = cast(dict, function).get("name")
|
| 1720 |
+
if _name is not None and isinstance(_name, str):
|
| 1721 |
+
tool_set.add(_name)
|
| 1722 |
+
return list(tool_set)
|
| 1723 |
+
|
| 1724 |
+
def apply_tool_call_transformation_if_needed(
|
| 1725 |
+
self,
|
| 1726 |
+
message: Message,
|
| 1727 |
+
tools: Optional[List[ToolBlock]] = None,
|
| 1728 |
+
initial_finish_reason: Optional[str] = None,
|
| 1729 |
+
) -> Tuple[Message, Optional[str]]:
|
| 1730 |
+
"""
|
| 1731 |
+
Apply tool call transformation to a message.
|
| 1732 |
+
|
| 1733 |
+
LLM providers (e.g. Bedrock, Vertex AI) sometimes return tool call in the response content.
|
| 1734 |
+
|
| 1735 |
+
If the response content is a JSON object, we can parse it and return the tool call in the tool_calls field.
|
| 1736 |
+
"""
|
| 1737 |
+
returned_finish_reason = initial_finish_reason
|
| 1738 |
+
if tools is None:
|
| 1739 |
+
return message, returned_finish_reason
|
| 1740 |
+
|
| 1741 |
+
if message.content is not None:
|
| 1742 |
+
try:
|
| 1743 |
+
tool_call_names = self.get_tool_call_names(tools)
|
| 1744 |
+
json_content = json.loads(message.content)
|
| 1745 |
+
if (
|
| 1746 |
+
json_content.get("type") == "function"
|
| 1747 |
+
and json_content.get("name") in tool_call_names
|
| 1748 |
+
):
|
| 1749 |
+
tool_calls = [
|
| 1750 |
+
ChatCompletionMessageToolCall(function=Function(**json_content))
|
| 1751 |
+
]
|
| 1752 |
+
|
| 1753 |
+
message.tool_calls = tool_calls
|
| 1754 |
+
message.content = None
|
| 1755 |
+
returned_finish_reason = "tool_calls"
|
| 1756 |
+
except Exception:
|
| 1757 |
+
pass
|
| 1758 |
+
|
| 1759 |
+
return message, returned_finish_reason
|
| 1760 |
+
|
| 1761 |
+
def _translate_message_content(
|
| 1762 |
+
self, content_blocks: List[ContentBlock]
|
| 1763 |
+
) -> Tuple[
|
| 1764 |
+
str,
|
| 1765 |
+
List[ChatCompletionToolCallChunk],
|
| 1766 |
+
Optional[List[BedrockConverseReasoningContentBlock]],
|
| 1767 |
+
Optional[List[CitationsContentBlock]],
|
| 1768 |
+
]:
|
| 1769 |
+
"""
|
| 1770 |
+
Translate the message content to a string and a list of tool calls, reasoning content blocks, and citations.
|
| 1771 |
+
|
| 1772 |
+
Returns:
|
| 1773 |
+
content_str: str
|
| 1774 |
+
tools: List[ChatCompletionToolCallChunk]
|
| 1775 |
+
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]]
|
| 1776 |
+
citationsContentBlocks: Optional[List[CitationsContentBlock]] - Citations from Nova grounding
|
| 1777 |
+
"""
|
| 1778 |
+
content_str = ""
|
| 1779 |
+
tools: List[ChatCompletionToolCallChunk] = []
|
| 1780 |
+
reasoningContentBlocks: Optional[
|
| 1781 |
+
List[BedrockConverseReasoningContentBlock]
|
| 1782 |
+
] = None
|
| 1783 |
+
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
|
| 1784 |
+
for idx, content in enumerate(content_blocks):
|
| 1785 |
+
"""
|
| 1786 |
+
- Content is either a tool response or text
|
| 1787 |
+
"""
|
| 1788 |
+
extracted_reasoning_content_str: Optional[str] = None
|
| 1789 |
+
if "text" in content:
|
| 1790 |
+
(
|
| 1791 |
+
extracted_reasoning_content_str,
|
| 1792 |
+
_content_str,
|
| 1793 |
+
) = _parse_content_for_reasoning(content["text"])
|
| 1794 |
+
if _content_str is not None:
|
| 1795 |
+
content_str += _content_str
|
| 1796 |
+
if "toolUse" in content:
|
| 1797 |
+
## check tool name was formatted by litellm
|
| 1798 |
+
_response_tool_name = content["toolUse"]["name"]
|
| 1799 |
+
response_tool_name = get_bedrock_tool_name(
|
| 1800 |
+
response_tool_name=_response_tool_name
|
| 1801 |
+
)
|
| 1802 |
+
_function_chunk = ChatCompletionToolCallFunctionChunk(
|
| 1803 |
+
name=response_tool_name,
|
| 1804 |
+
arguments=json.dumps(content["toolUse"]["input"]),
|
| 1805 |
+
)
|
| 1806 |
+
|
| 1807 |
+
_tool_response_chunk = ChatCompletionToolCallChunk(
|
| 1808 |
+
id=content["toolUse"]["toolUseId"],
|
| 1809 |
+
type="function",
|
| 1810 |
+
function=_function_chunk,
|
| 1811 |
+
index=idx,
|
| 1812 |
+
)
|
| 1813 |
+
tools.append(_tool_response_chunk)
|
| 1814 |
+
if extracted_reasoning_content_str is not None:
|
| 1815 |
+
if reasoningContentBlocks is None:
|
| 1816 |
+
reasoningContentBlocks = []
|
| 1817 |
+
reasoningContentBlocks.append(
|
| 1818 |
+
BedrockConverseReasoningContentBlock(
|
| 1819 |
+
reasoningText=BedrockConverseReasoningTextBlock(
|
| 1820 |
+
text=extracted_reasoning_content_str,
|
| 1821 |
+
)
|
| 1822 |
+
)
|
| 1823 |
+
)
|
| 1824 |
+
if "reasoningContent" in content:
|
| 1825 |
+
if reasoningContentBlocks is None:
|
| 1826 |
+
reasoningContentBlocks = []
|
| 1827 |
+
reasoningContentBlocks.append(content["reasoningContent"])
|
| 1828 |
+
# Handle Nova grounding citations content
|
| 1829 |
+
if "citationsContent" in content:
|
| 1830 |
+
if citationsContentBlocks is None:
|
| 1831 |
+
citationsContentBlocks = []
|
| 1832 |
+
citationsContentBlocks.append(content["citationsContent"])
|
| 1833 |
+
|
| 1834 |
+
return content_str, tools, reasoningContentBlocks, citationsContentBlocks
|
| 1835 |
+
|
| 1836 |
+
@staticmethod
|
| 1837 |
+
def _unwrap_bedrock_properties(json_str: str) -> str:
|
| 1838 |
+
"""
|
| 1839 |
+
Unwrap Bedrock's response_format JSON structure.
|
| 1840 |
+
|
| 1841 |
+
If the JSON has a single "properties" key, extract its value.
|
| 1842 |
+
Otherwise, return the original string.
|
| 1843 |
+
|
| 1844 |
+
Args:
|
| 1845 |
+
json_str: JSON string to unwrap
|
| 1846 |
+
|
| 1847 |
+
Returns:
|
| 1848 |
+
Unwrapped JSON string or original if unwrapping not needed
|
| 1849 |
+
"""
|
| 1850 |
+
try:
|
| 1851 |
+
response_data = json.loads(json_str)
|
| 1852 |
+
if (
|
| 1853 |
+
isinstance(response_data, dict)
|
| 1854 |
+
and "properties" in response_data
|
| 1855 |
+
and len(response_data) == 1
|
| 1856 |
+
):
|
| 1857 |
+
response_data = response_data["properties"]
|
| 1858 |
+
return json.dumps(response_data)
|
| 1859 |
+
except json.JSONDecodeError:
|
| 1860 |
+
pass
|
| 1861 |
+
return json_str
|
| 1862 |
+
|
| 1863 |
+
@staticmethod
|
| 1864 |
+
def _filter_json_mode_tools(
|
| 1865 |
+
json_mode: Optional[bool],
|
| 1866 |
+
tools: List[ChatCompletionToolCallChunk],
|
| 1867 |
+
chat_completion_message: ChatCompletionResponseMessage,
|
| 1868 |
+
) -> Optional[List[ChatCompletionToolCallChunk]]:
|
| 1869 |
+
"""
|
| 1870 |
+
When json_mode is True, Bedrock may return the internal `json_tool_call`
|
| 1871 |
+
tool alongside real user-defined tools. This method handles 3 scenarios:
|
| 1872 |
+
|
| 1873 |
+
1. Only json_tool_call present -> convert to text content, return None
|
| 1874 |
+
2. Mixed json_tool_call + real -> filter out json_tool_call, return real tools
|
| 1875 |
+
3. No json_tool_call / no json_mode -> return tools as-is
|
| 1876 |
+
"""
|
| 1877 |
+
if not json_mode or not tools:
|
| 1878 |
+
return tools if tools else None
|
| 1879 |
+
|
| 1880 |
+
json_tool_indices = [
|
| 1881 |
+
i
|
| 1882 |
+
for i, t in enumerate(tools)
|
| 1883 |
+
if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME
|
| 1884 |
+
]
|
| 1885 |
+
|
| 1886 |
+
if not json_tool_indices:
|
| 1887 |
+
# No json_tool_call found, return tools unchanged
|
| 1888 |
+
return tools
|
| 1889 |
+
|
| 1890 |
+
if len(json_tool_indices) == len(tools):
|
| 1891 |
+
# All tools are json_tool_call — convert first one to content
|
| 1892 |
+
verbose_logger.debug(
|
| 1893 |
+
"Processing JSON tool call response for response_format"
|
| 1894 |
+
)
|
| 1895 |
+
json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments")
|
| 1896 |
+
if json_mode_content_str is not None:
|
| 1897 |
+
json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(
|
| 1898 |
+
json_mode_content_str
|
| 1899 |
+
)
|
| 1900 |
+
chat_completion_message["content"] = json_mode_content_str
|
| 1901 |
+
return None
|
| 1902 |
+
|
| 1903 |
+
# Mixed: filter out json_tool_call, keep real tools.
|
| 1904 |
+
# Preserve the json_tool_call content as message text so the structured
|
| 1905 |
+
# output from response_format is not silently lost.
|
| 1906 |
+
first_idx = json_tool_indices[0]
|
| 1907 |
+
json_mode_args = tools[first_idx]["function"].get("arguments")
|
| 1908 |
+
if json_mode_args is not None:
|
| 1909 |
+
json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(
|
| 1910 |
+
json_mode_args
|
| 1911 |
+
)
|
| 1912 |
+
existing = chat_completion_message.get("content") or ""
|
| 1913 |
+
chat_completion_message["content"] = (
|
| 1914 |
+
existing + json_mode_args if existing else json_mode_args
|
| 1915 |
+
)
|
| 1916 |
+
|
| 1917 |
+
real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices]
|
| 1918 |
+
return real_tools if real_tools else None
|
| 1919 |
+
|
| 1920 |
+
def _transform_response( # noqa: PLR0915
|
| 1921 |
+
self,
|
| 1922 |
+
model: str,
|
| 1923 |
+
response: httpx.Response,
|
| 1924 |
+
model_response: ModelResponse,
|
| 1925 |
+
stream: bool,
|
| 1926 |
+
logging_obj: Optional[Logging],
|
| 1927 |
+
optional_params: dict,
|
| 1928 |
+
api_key: Optional[str],
|
| 1929 |
+
data: Union[dict, str],
|
| 1930 |
+
messages: List,
|
| 1931 |
+
encoding,
|
| 1932 |
+
) -> ModelResponse:
|
| 1933 |
+
## LOGGING
|
| 1934 |
+
if logging_obj is not None:
|
| 1935 |
+
logging_obj.post_call(
|
| 1936 |
+
input=messages,
|
| 1937 |
+
api_key=api_key,
|
| 1938 |
+
original_response=response.text,
|
| 1939 |
+
additional_args={"complete_input_dict": data},
|
| 1940 |
+
)
|
| 1941 |
+
|
| 1942 |
+
json_mode: Optional[bool] = optional_params.get("json_mode", None)
|
| 1943 |
+
## RESPONSE OBJECT
|
| 1944 |
+
try:
|
| 1945 |
+
completion_response = ConverseResponseBlock(**response.json()) # type: ignore
|
| 1946 |
+
except Exception as e:
|
| 1947 |
+
raise BedrockError(
|
| 1948 |
+
message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
|
| 1949 |
+
response.text, str(e)
|
| 1950 |
+
),
|
| 1951 |
+
status_code=422,
|
| 1952 |
+
)
|
| 1953 |
+
|
| 1954 |
+
"""
|
| 1955 |
+
Bedrock Response Object has optional message block
|
| 1956 |
+
|
| 1957 |
+
completion_response["output"].get("message", None)
|
| 1958 |
+
|
| 1959 |
+
A message block looks like this (Example 1):
|
| 1960 |
+
"output": {
|
| 1961 |
+
"message": {
|
| 1962 |
+
"role": "assistant",
|
| 1963 |
+
"content": [
|
| 1964 |
+
{
|
| 1965 |
+
"text": "Is there anything else you'd like to talk about? Perhaps I can help with some economic questions or provide some information about economic concepts?"
|
| 1966 |
+
}
|
| 1967 |
+
]
|
| 1968 |
+
}
|
| 1969 |
+
},
|
| 1970 |
+
(Example 2):
|
| 1971 |
+
"output": {
|
| 1972 |
+
"message": {
|
| 1973 |
+
"role": "assistant",
|
| 1974 |
+
"content": [
|
| 1975 |
+
{
|
| 1976 |
+
"toolUse": {
|
| 1977 |
+
"toolUseId": "tooluse_hbTgdi0CSLq_hM4P8csZJA",
|
| 1978 |
+
"name": "top_song",
|
| 1979 |
+
"input": {
|
| 1980 |
+
"sign": "WZPZ"
|
| 1981 |
+
}
|
| 1982 |
+
}
|
| 1983 |
+
}
|
| 1984 |
+
]
|
| 1985 |
+
}
|
| 1986 |
+
}
|
| 1987 |
+
|
| 1988 |
+
"""
|
| 1989 |
+
message: Optional[MessageBlock] = completion_response["output"]["message"]
|
| 1990 |
+
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
|
| 1991 |
+
content_str = ""
|
| 1992 |
+
tools: List[ChatCompletionToolCallChunk] = []
|
| 1993 |
+
reasoningContentBlocks: Optional[
|
| 1994 |
+
List[BedrockConverseReasoningContentBlock]
|
| 1995 |
+
] = None
|
| 1996 |
+
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
|
| 1997 |
+
|
| 1998 |
+
if message is not None:
|
| 1999 |
+
(
|
| 2000 |
+
content_str,
|
| 2001 |
+
tools,
|
| 2002 |
+
reasoningContentBlocks,
|
| 2003 |
+
citationsContentBlocks,
|
| 2004 |
+
) = self._translate_message_content(message["content"])
|
| 2005 |
+
|
| 2006 |
+
# Initialize provider_specific_fields if we have any special content blocks
|
| 2007 |
+
provider_specific_fields: dict = {}
|
| 2008 |
+
if reasoningContentBlocks is not None:
|
| 2009 |
+
provider_specific_fields["reasoningContentBlocks"] = reasoningContentBlocks
|
| 2010 |
+
if citationsContentBlocks is not None:
|
| 2011 |
+
provider_specific_fields["citationsContent"] = citationsContentBlocks
|
| 2012 |
+
|
| 2013 |
+
if provider_specific_fields:
|
| 2014 |
+
chat_completion_message[
|
| 2015 |
+
"provider_specific_fields"
|
| 2016 |
+
] = provider_specific_fields
|
| 2017 |
+
|
| 2018 |
+
if reasoningContentBlocks is not None:
|
| 2019 |
+
chat_completion_message[
|
| 2020 |
+
"reasoning_content"
|
| 2021 |
+
] = self._transform_reasoning_content(reasoningContentBlocks)
|
| 2022 |
+
chat_completion_message[
|
| 2023 |
+
"thinking_blocks"
|
| 2024 |
+
] = self._transform_thinking_blocks(reasoningContentBlocks)
|
| 2025 |
+
chat_completion_message["content"] = content_str
|
| 2026 |
+
filtered_tools = self._filter_json_mode_tools(
|
| 2027 |
+
json_mode=json_mode,
|
| 2028 |
+
tools=tools,
|
| 2029 |
+
chat_completion_message=chat_completion_message,
|
| 2030 |
+
)
|
| 2031 |
+
if filtered_tools:
|
| 2032 |
+
chat_completion_message["tool_calls"] = filtered_tools
|
| 2033 |
+
|
| 2034 |
+
## CALCULATING USAGE - bedrock returns usage in the headers
|
| 2035 |
+
usage = self._transform_usage(
|
| 2036 |
+
completion_response["usage"],
|
| 2037 |
+
reasoning_content=chat_completion_message.get("reasoning_content"),
|
| 2038 |
+
)
|
| 2039 |
+
|
| 2040 |
+
## HANDLE TOOL CALLS
|
| 2041 |
+
_message = Message(**chat_completion_message)
|
| 2042 |
+
initial_finish_reason = map_finish_reason(completion_response["stopReason"])
|
| 2043 |
+
|
| 2044 |
+
(
|
| 2045 |
+
returned_message,
|
| 2046 |
+
returned_finish_reason,
|
| 2047 |
+
) = self.apply_tool_call_transformation_if_needed(
|
| 2048 |
+
message=_message,
|
| 2049 |
+
tools=optional_params.get("tools"),
|
| 2050 |
+
initial_finish_reason=initial_finish_reason,
|
| 2051 |
+
)
|
| 2052 |
+
model_response.choices = [
|
| 2053 |
+
litellm.Choices(
|
| 2054 |
+
finish_reason=returned_finish_reason,
|
| 2055 |
+
index=0,
|
| 2056 |
+
message=returned_message,
|
| 2057 |
+
)
|
| 2058 |
+
]
|
| 2059 |
+
model_response.created = int(time.time())
|
| 2060 |
+
model_response.model = model
|
| 2061 |
+
|
| 2062 |
+
setattr(model_response, "usage", usage)
|
| 2063 |
+
|
| 2064 |
+
# Add "trace" from Bedrock guardrails - if user has opted in to returning it
|
| 2065 |
+
if "trace" in completion_response:
|
| 2066 |
+
setattr(model_response, "trace", completion_response["trace"])
|
| 2067 |
+
|
| 2068 |
+
# Add service_tier if present in Bedrock response
|
| 2069 |
+
# Map Bedrock serviceTier (object) to OpenAI service_tier (string)
|
| 2070 |
+
if "serviceTier" in completion_response:
|
| 2071 |
+
service_tier_block = completion_response["serviceTier"]
|
| 2072 |
+
if isinstance(service_tier_block, dict) and "type" in service_tier_block:
|
| 2073 |
+
setattr(model_response, "service_tier", service_tier_block["type"])
|
| 2074 |
+
|
| 2075 |
+
return model_response
|
| 2076 |
+
|
| 2077 |
+
def get_error_class(
|
| 2078 |
+
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
| 2079 |
+
) -> BaseLLMException:
|
| 2080 |
+
return BedrockError(
|
| 2081 |
+
message=error_message,
|
| 2082 |
+
status_code=status_code,
|
| 2083 |
+
headers=headers,
|
| 2084 |
+
)
|
| 2085 |
+
|
| 2086 |
+
def validate_environment(
|
| 2087 |
+
self,
|
| 2088 |
+
headers: dict,
|
| 2089 |
+
model: str,
|
| 2090 |
+
messages: List[AllMessageValues],
|
| 2091 |
+
optional_params: dict,
|
| 2092 |
+
litellm_params: dict,
|
| 2093 |
+
api_key: Optional[str] = None,
|
| 2094 |
+
api_base: Optional[str] = None,
|
| 2095 |
+
) -> dict:
|
| 2096 |
+
if api_key:
|
| 2097 |
+
headers["Authorization"] = f"Bearer {api_key}"
|
| 2098 |
+
return headers
|
| 2099 |
+
|
| 2100 |
+
def should_fake_stream(
|
| 2101 |
+
self,
|
| 2102 |
+
model: Optional[str],
|
| 2103 |
+
stream: Optional[bool],
|
| 2104 |
+
custom_llm_provider: Optional[str] = None,
|
| 2105 |
+
fake_stream: Optional[bool] = None,
|
| 2106 |
+
) -> bool:
|
| 2107 |
+
"""
|
| 2108 |
+
Returns True if the model/provider should fake stream
|
| 2109 |
+
"""
|
| 2110 |
+
###################################################################
|
| 2111 |
+
# If an upstream method already set fake_stream to True, return True
|
| 2112 |
+
###################################################################
|
| 2113 |
+
if fake_stream is True:
|
| 2114 |
+
return True
|
| 2115 |
+
|
| 2116 |
+
###################################################################
|
| 2117 |
+
# Bedrock Converse Specific Logic
|
| 2118 |
+
###################################################################
|
| 2119 |
+
if stream is True:
|
| 2120 |
+
if model is not None:
|
| 2121 |
+
###################################################################
|
| 2122 |
+
# AI21 models do not support streaming
|
| 2123 |
+
###################################################################
|
| 2124 |
+
if "ai21" in model:
|
| 2125 |
+
return True
|
| 2126 |
+
return False
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_agent/transformation.py
ADDED
|
@@ -0,0 +1,547 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation for Bedrock Invoke Agent
|
| 3 |
+
|
| 4 |
+
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import base64
|
| 8 |
+
import json
|
| 9 |
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
| 10 |
+
|
| 11 |
+
import httpx
|
| 12 |
+
|
| 13 |
+
from litellm._logging import verbose_logger
|
| 14 |
+
from litellm._uuid import uuid
|
| 15 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 16 |
+
convert_content_list_to_str,
|
| 17 |
+
)
|
| 18 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
| 19 |
+
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
| 20 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 21 |
+
from litellm.types.llms.bedrock_invoke_agents import (
|
| 22 |
+
InvokeAgentChunkPayload,
|
| 23 |
+
InvokeAgentEvent,
|
| 24 |
+
InvokeAgentEventHeaders,
|
| 25 |
+
InvokeAgentEventList,
|
| 26 |
+
InvokeAgentMetadata,
|
| 27 |
+
InvokeAgentModelInvocationInput,
|
| 28 |
+
InvokeAgentModelInvocationOutput,
|
| 29 |
+
InvokeAgentOrchestrationTrace,
|
| 30 |
+
InvokeAgentPreProcessingTrace,
|
| 31 |
+
InvokeAgentTrace,
|
| 32 |
+
InvokeAgentTracePayload,
|
| 33 |
+
InvokeAgentUsage,
|
| 34 |
+
)
|
| 35 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 36 |
+
from litellm.types.utils import Choices, Message, ModelResponse
|
| 37 |
+
|
| 38 |
+
if TYPE_CHECKING:
|
| 39 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 40 |
+
|
| 41 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 42 |
+
else:
|
| 43 |
+
LiteLLMLoggingObj = Any
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
|
| 47 |
+
def __init__(self, **kwargs):
|
| 48 |
+
BaseConfig.__init__(self, **kwargs)
|
| 49 |
+
BaseAWSLLM.__init__(self, **kwargs)
|
| 50 |
+
|
| 51 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 52 |
+
"""
|
| 53 |
+
This is a base invoke agent model mapping. For Invoke Agent - define a bedrock provider specific config that extends this class.
|
| 54 |
+
|
| 55 |
+
Bedrock Invoke Agents has 0 OpenAI compatible params
|
| 56 |
+
|
| 57 |
+
As of May 29th, 2025 - they don't support streaming.
|
| 58 |
+
"""
|
| 59 |
+
return []
|
| 60 |
+
|
| 61 |
+
def map_openai_params(
|
| 62 |
+
self,
|
| 63 |
+
non_default_params: dict,
|
| 64 |
+
optional_params: dict,
|
| 65 |
+
model: str,
|
| 66 |
+
drop_params: bool,
|
| 67 |
+
) -> dict:
|
| 68 |
+
"""
|
| 69 |
+
This is a base invoke agent model mapping. For Invoke Agent - define a bedrock provider specific config that extends this class.
|
| 70 |
+
"""
|
| 71 |
+
return optional_params
|
| 72 |
+
|
| 73 |
+
def get_complete_url(
|
| 74 |
+
self,
|
| 75 |
+
api_base: Optional[str],
|
| 76 |
+
api_key: Optional[str],
|
| 77 |
+
model: str,
|
| 78 |
+
optional_params: dict,
|
| 79 |
+
litellm_params: dict,
|
| 80 |
+
stream: Optional[bool] = None,
|
| 81 |
+
) -> str:
|
| 82 |
+
"""
|
| 83 |
+
Get the complete url for the request
|
| 84 |
+
"""
|
| 85 |
+
### SET RUNTIME ENDPOINT ###
|
| 86 |
+
aws_bedrock_runtime_endpoint = optional_params.get(
|
| 87 |
+
"aws_bedrock_runtime_endpoint", None
|
| 88 |
+
) # https://bedrock-runtime.{region_name}.amazonaws.com
|
| 89 |
+
endpoint_url, _ = self.get_runtime_endpoint(
|
| 90 |
+
api_base=api_base,
|
| 91 |
+
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
| 92 |
+
aws_region_name=self._get_aws_region_name(
|
| 93 |
+
optional_params=optional_params, model=model
|
| 94 |
+
),
|
| 95 |
+
endpoint_type="agent",
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
agent_id, agent_alias_id = self._get_agent_id_and_alias_id(model)
|
| 99 |
+
session_id = self._get_session_id(optional_params)
|
| 100 |
+
|
| 101 |
+
endpoint_url = f"{endpoint_url}/agents/{agent_id}/agentAliases/{agent_alias_id}/sessions/{session_id}/text"
|
| 102 |
+
|
| 103 |
+
return endpoint_url
|
| 104 |
+
|
| 105 |
+
def sign_request(
|
| 106 |
+
self,
|
| 107 |
+
headers: dict,
|
| 108 |
+
optional_params: dict,
|
| 109 |
+
request_data: dict,
|
| 110 |
+
api_base: str,
|
| 111 |
+
api_key: Optional[str] = None,
|
| 112 |
+
model: Optional[str] = None,
|
| 113 |
+
stream: Optional[bool] = None,
|
| 114 |
+
fake_stream: Optional[bool] = None,
|
| 115 |
+
) -> Tuple[dict, Optional[bytes]]:
|
| 116 |
+
return self._sign_request(
|
| 117 |
+
service_name="bedrock",
|
| 118 |
+
headers=headers,
|
| 119 |
+
optional_params=optional_params,
|
| 120 |
+
request_data=request_data,
|
| 121 |
+
api_base=api_base,
|
| 122 |
+
model=model,
|
| 123 |
+
stream=stream,
|
| 124 |
+
fake_stream=fake_stream,
|
| 125 |
+
api_key=api_key,
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
def _get_agent_id_and_alias_id(self, model: str) -> tuple[str, str]:
|
| 129 |
+
"""
|
| 130 |
+
model = "agent/L1RT58GYRW/MFPSBCXYTW"
|
| 131 |
+
agent_id = "L1RT58GYRW"
|
| 132 |
+
agent_alias_id = "MFPSBCXYTW"
|
| 133 |
+
"""
|
| 134 |
+
# Split the model string by '/' and extract components
|
| 135 |
+
parts = model.split("/")
|
| 136 |
+
if len(parts) != 3 or parts[0] != "agent":
|
| 137 |
+
raise ValueError(
|
| 138 |
+
"Invalid model format. Expected format: 'model=agent/AGENT_ID/ALIAS_ID'"
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
return parts[1], parts[2] # Return (agent_id, agent_alias_id)
|
| 142 |
+
|
| 143 |
+
def _get_session_id(self, optional_params: dict) -> str:
|
| 144 |
+
""" """
|
| 145 |
+
return optional_params.get("sessionID", None) or str(uuid.uuid4())
|
| 146 |
+
|
| 147 |
+
def transform_request(
|
| 148 |
+
self,
|
| 149 |
+
model: str,
|
| 150 |
+
messages: List[AllMessageValues],
|
| 151 |
+
optional_params: dict,
|
| 152 |
+
litellm_params: dict,
|
| 153 |
+
headers: dict,
|
| 154 |
+
) -> dict:
|
| 155 |
+
# use the last message content as the query
|
| 156 |
+
query: str = convert_content_list_to_str(messages[-1])
|
| 157 |
+
return {
|
| 158 |
+
"inputText": query,
|
| 159 |
+
"enableTrace": True,
|
| 160 |
+
**optional_params,
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
def _parse_aws_event_stream(self, raw_content: bytes) -> InvokeAgentEventList:
|
| 164 |
+
"""
|
| 165 |
+
Parse AWS event stream format using boto3/botocore's built-in parser.
|
| 166 |
+
This is the same approach used in the existing AWSEventStreamDecoder.
|
| 167 |
+
"""
|
| 168 |
+
try:
|
| 169 |
+
from botocore.eventstream import EventStreamBuffer
|
| 170 |
+
from botocore.parsers import EventStreamJSONParser
|
| 171 |
+
except ImportError:
|
| 172 |
+
raise ImportError("boto3/botocore is required for AWS event stream parsing")
|
| 173 |
+
|
| 174 |
+
events: InvokeAgentEventList = []
|
| 175 |
+
parser = EventStreamJSONParser()
|
| 176 |
+
event_stream_buffer = EventStreamBuffer()
|
| 177 |
+
|
| 178 |
+
# Add the entire response to the buffer
|
| 179 |
+
event_stream_buffer.add_data(raw_content)
|
| 180 |
+
|
| 181 |
+
# Process all events in the buffer
|
| 182 |
+
for event in event_stream_buffer:
|
| 183 |
+
try:
|
| 184 |
+
headers = self._extract_headers_from_event(event)
|
| 185 |
+
|
| 186 |
+
event_type = headers.get("event_type", "")
|
| 187 |
+
|
| 188 |
+
if event_type == "chunk":
|
| 189 |
+
# Handle chunk events specially - they contain decoded content, not JSON
|
| 190 |
+
message = self._parse_message_from_event(event, parser)
|
| 191 |
+
parsed_event: InvokeAgentEvent = InvokeAgentEvent()
|
| 192 |
+
if message:
|
| 193 |
+
# For chunk events, create a payload with the decoded content
|
| 194 |
+
parsed_event = {
|
| 195 |
+
"headers": headers,
|
| 196 |
+
"payload": {
|
| 197 |
+
"bytes": base64.b64encode(
|
| 198 |
+
message.encode("utf-8")
|
| 199 |
+
).decode("utf-8")
|
| 200 |
+
}, # Re-encode for consistency
|
| 201 |
+
}
|
| 202 |
+
events.append(parsed_event)
|
| 203 |
+
|
| 204 |
+
elif event_type == "trace":
|
| 205 |
+
# Handle trace events normally - they contain JSON
|
| 206 |
+
message = self._parse_message_from_event(event, parser)
|
| 207 |
+
|
| 208 |
+
if message:
|
| 209 |
+
try:
|
| 210 |
+
event_data = json.loads(message)
|
| 211 |
+
parsed_event = {
|
| 212 |
+
"headers": headers,
|
| 213 |
+
"payload": event_data,
|
| 214 |
+
}
|
| 215 |
+
events.append(parsed_event)
|
| 216 |
+
except json.JSONDecodeError as e:
|
| 217 |
+
verbose_logger.warning(
|
| 218 |
+
f"Failed to parse trace event JSON: {e}"
|
| 219 |
+
)
|
| 220 |
+
else:
|
| 221 |
+
verbose_logger.debug(f"Unknown event type: {event_type}")
|
| 222 |
+
|
| 223 |
+
except Exception as e:
|
| 224 |
+
verbose_logger.error(f"Error processing event: {e}")
|
| 225 |
+
continue
|
| 226 |
+
|
| 227 |
+
return events
|
| 228 |
+
|
| 229 |
+
def _parse_message_from_event(self, event, parser) -> Optional[str]:
|
| 230 |
+
"""Extract message content from an AWS event, adapted from AWSEventStreamDecoder."""
|
| 231 |
+
try:
|
| 232 |
+
response_dict = event.to_response_dict()
|
| 233 |
+
verbose_logger.debug(f"Response dict: {response_dict}")
|
| 234 |
+
|
| 235 |
+
# Use the same response shape parsing as the existing decoder
|
| 236 |
+
parsed_response = parser.parse(
|
| 237 |
+
response_dict, self._get_response_stream_shape()
|
| 238 |
+
)
|
| 239 |
+
verbose_logger.debug(f"Parsed response: {parsed_response}")
|
| 240 |
+
|
| 241 |
+
if response_dict["status_code"] != 200:
|
| 242 |
+
decoded_body = response_dict["body"].decode()
|
| 243 |
+
if isinstance(decoded_body, dict):
|
| 244 |
+
error_message = decoded_body.get("message")
|
| 245 |
+
elif isinstance(decoded_body, str):
|
| 246 |
+
error_message = decoded_body
|
| 247 |
+
else:
|
| 248 |
+
error_message = ""
|
| 249 |
+
exception_status = response_dict["headers"].get(":exception-type")
|
| 250 |
+
error_message = exception_status + " " + error_message
|
| 251 |
+
raise BedrockError(
|
| 252 |
+
status_code=response_dict["status_code"],
|
| 253 |
+
message=(
|
| 254 |
+
json.dumps(error_message)
|
| 255 |
+
if isinstance(error_message, dict)
|
| 256 |
+
else error_message
|
| 257 |
+
),
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
if "chunk" in parsed_response:
|
| 261 |
+
chunk = parsed_response.get("chunk")
|
| 262 |
+
if not chunk:
|
| 263 |
+
return None
|
| 264 |
+
return chunk.get("bytes").decode()
|
| 265 |
+
else:
|
| 266 |
+
chunk = response_dict.get("body")
|
| 267 |
+
if not chunk:
|
| 268 |
+
return None
|
| 269 |
+
return chunk.decode()
|
| 270 |
+
|
| 271 |
+
except Exception as e:
|
| 272 |
+
verbose_logger.debug(f"Error parsing message from event: {e}")
|
| 273 |
+
return None
|
| 274 |
+
|
| 275 |
+
def _extract_headers_from_event(self, event) -> InvokeAgentEventHeaders:
|
| 276 |
+
"""Extract headers from an AWS event for categorization."""
|
| 277 |
+
try:
|
| 278 |
+
response_dict = event.to_response_dict()
|
| 279 |
+
headers = response_dict.get("headers", {})
|
| 280 |
+
|
| 281 |
+
# Extract the event-type and content-type headers that we care about
|
| 282 |
+
return InvokeAgentEventHeaders(
|
| 283 |
+
event_type=headers.get(":event-type", ""),
|
| 284 |
+
content_type=headers.get(":content-type", ""),
|
| 285 |
+
message_type=headers.get(":message-type", ""),
|
| 286 |
+
)
|
| 287 |
+
except Exception as e:
|
| 288 |
+
verbose_logger.debug(f"Error extracting headers: {e}")
|
| 289 |
+
return InvokeAgentEventHeaders(
|
| 290 |
+
event_type="", content_type="", message_type=""
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
+
def _get_response_stream_shape(self):
|
| 294 |
+
"""Get the response stream shape for parsing, reusing existing logic."""
|
| 295 |
+
try:
|
| 296 |
+
# Try to reuse the cached shape from the existing decoder
|
| 297 |
+
from litellm.llms.bedrock.chat.invoke_handler import (
|
| 298 |
+
get_response_stream_shape,
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
return get_response_stream_shape()
|
| 302 |
+
except ImportError:
|
| 303 |
+
# Fallback: create our own shape
|
| 304 |
+
try:
|
| 305 |
+
from botocore.loaders import Loader
|
| 306 |
+
from botocore.model import ServiceModel
|
| 307 |
+
|
| 308 |
+
loader = Loader()
|
| 309 |
+
bedrock_service_dict = loader.load_service_model(
|
| 310 |
+
"bedrock-runtime", "service-2"
|
| 311 |
+
)
|
| 312 |
+
bedrock_service_model = ServiceModel(bedrock_service_dict)
|
| 313 |
+
return bedrock_service_model.shape_for("ResponseStream")
|
| 314 |
+
except Exception as e:
|
| 315 |
+
verbose_logger.warning(f"Could not load response stream shape: {e}")
|
| 316 |
+
return None
|
| 317 |
+
|
| 318 |
+
def _extract_response_content(self, events: InvokeAgentEventList) -> str:
|
| 319 |
+
"""Extract the final response content from parsed events."""
|
| 320 |
+
response_parts = []
|
| 321 |
+
|
| 322 |
+
for event in events:
|
| 323 |
+
headers = event.get("headers", {})
|
| 324 |
+
payload = event.get("payload")
|
| 325 |
+
|
| 326 |
+
event_type = headers.get(
|
| 327 |
+
"event_type"
|
| 328 |
+
) # Note: using event_type not event-type
|
| 329 |
+
|
| 330 |
+
if event_type == "chunk" and payload:
|
| 331 |
+
# Extract base64 encoded content from chunk events
|
| 332 |
+
chunk_payload: InvokeAgentChunkPayload = payload # type: ignore
|
| 333 |
+
encoded_bytes = chunk_payload.get("bytes", "")
|
| 334 |
+
if encoded_bytes:
|
| 335 |
+
try:
|
| 336 |
+
decoded_content = base64.b64decode(encoded_bytes).decode(
|
| 337 |
+
"utf-8"
|
| 338 |
+
)
|
| 339 |
+
response_parts.append(decoded_content)
|
| 340 |
+
except Exception as e:
|
| 341 |
+
verbose_logger.warning(f"Failed to decode chunk content: {e}")
|
| 342 |
+
|
| 343 |
+
return "".join(response_parts)
|
| 344 |
+
|
| 345 |
+
def _extract_usage_info(self, events: InvokeAgentEventList) -> InvokeAgentUsage:
|
| 346 |
+
"""Extract token usage information from trace events."""
|
| 347 |
+
usage_info = InvokeAgentUsage(
|
| 348 |
+
inputTokens=0,
|
| 349 |
+
outputTokens=0,
|
| 350 |
+
model=None,
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
response_model: Optional[str] = None
|
| 354 |
+
|
| 355 |
+
for event in events:
|
| 356 |
+
if not self._is_trace_event(event):
|
| 357 |
+
continue
|
| 358 |
+
|
| 359 |
+
trace_data = self._get_trace_data(event)
|
| 360 |
+
if not trace_data:
|
| 361 |
+
continue
|
| 362 |
+
|
| 363 |
+
verbose_logger.debug(f"Trace event: {trace_data}")
|
| 364 |
+
|
| 365 |
+
# Extract usage from pre-processing trace
|
| 366 |
+
self._extract_and_update_preprocessing_usage(
|
| 367 |
+
trace_data=trace_data,
|
| 368 |
+
usage_info=usage_info,
|
| 369 |
+
)
|
| 370 |
+
|
| 371 |
+
# Extract model from orchestration trace
|
| 372 |
+
if response_model is None:
|
| 373 |
+
response_model = self._extract_orchestration_model(trace_data)
|
| 374 |
+
|
| 375 |
+
usage_info["model"] = response_model
|
| 376 |
+
return usage_info
|
| 377 |
+
|
| 378 |
+
def _is_trace_event(self, event: InvokeAgentEvent) -> bool:
|
| 379 |
+
"""Check if the event is a trace event."""
|
| 380 |
+
headers = event.get("headers", {})
|
| 381 |
+
event_type = headers.get("event_type")
|
| 382 |
+
payload = event.get("payload")
|
| 383 |
+
return event_type == "trace" and payload is not None
|
| 384 |
+
|
| 385 |
+
def _get_trace_data(self, event: InvokeAgentEvent) -> Optional[InvokeAgentTrace]:
|
| 386 |
+
"""Extract trace data from a trace event."""
|
| 387 |
+
payload = event.get("payload")
|
| 388 |
+
if not payload:
|
| 389 |
+
return None
|
| 390 |
+
|
| 391 |
+
trace_payload: InvokeAgentTracePayload = payload # type: ignore
|
| 392 |
+
return trace_payload.get("trace", {})
|
| 393 |
+
|
| 394 |
+
def _extract_and_update_preprocessing_usage(
|
| 395 |
+
self, trace_data: InvokeAgentTrace, usage_info: InvokeAgentUsage
|
| 396 |
+
) -> None:
|
| 397 |
+
"""Extract usage information from preprocessing trace."""
|
| 398 |
+
pre_processing: Optional[InvokeAgentPreProcessingTrace] = trace_data.get(
|
| 399 |
+
"preProcessingTrace"
|
| 400 |
+
)
|
| 401 |
+
if not pre_processing:
|
| 402 |
+
return
|
| 403 |
+
|
| 404 |
+
model_output: Optional[InvokeAgentModelInvocationOutput] = (
|
| 405 |
+
pre_processing.get("modelInvocationOutput")
|
| 406 |
+
or InvokeAgentModelInvocationOutput()
|
| 407 |
+
)
|
| 408 |
+
if not model_output:
|
| 409 |
+
return
|
| 410 |
+
|
| 411 |
+
metadata: Optional[InvokeAgentMetadata] = (
|
| 412 |
+
model_output.get("metadata") or InvokeAgentMetadata()
|
| 413 |
+
)
|
| 414 |
+
if not metadata:
|
| 415 |
+
return
|
| 416 |
+
|
| 417 |
+
usage: Optional[Union[InvokeAgentUsage, Dict]] = metadata.get("usage", {})
|
| 418 |
+
if not usage:
|
| 419 |
+
return
|
| 420 |
+
|
| 421 |
+
usage_info["inputTokens"] += usage.get("inputTokens", 0)
|
| 422 |
+
usage_info["outputTokens"] += usage.get("outputTokens", 0)
|
| 423 |
+
|
| 424 |
+
def _extract_orchestration_model(
|
| 425 |
+
self, trace_data: InvokeAgentTrace
|
| 426 |
+
) -> Optional[str]:
|
| 427 |
+
"""Extract model information from orchestration trace."""
|
| 428 |
+
orchestration_trace: Optional[InvokeAgentOrchestrationTrace] = trace_data.get(
|
| 429 |
+
"orchestrationTrace"
|
| 430 |
+
)
|
| 431 |
+
if not orchestration_trace:
|
| 432 |
+
return None
|
| 433 |
+
|
| 434 |
+
model_invocation: Optional[InvokeAgentModelInvocationInput] = (
|
| 435 |
+
orchestration_trace.get("modelInvocationInput")
|
| 436 |
+
or InvokeAgentModelInvocationInput()
|
| 437 |
+
)
|
| 438 |
+
if not model_invocation:
|
| 439 |
+
return None
|
| 440 |
+
|
| 441 |
+
return model_invocation.get("foundationModel")
|
| 442 |
+
|
| 443 |
+
def _build_model_response(
|
| 444 |
+
self,
|
| 445 |
+
content: str,
|
| 446 |
+
model: str,
|
| 447 |
+
usage_info: InvokeAgentUsage,
|
| 448 |
+
model_response: ModelResponse,
|
| 449 |
+
) -> ModelResponse:
|
| 450 |
+
"""Build the final ModelResponse object."""
|
| 451 |
+
|
| 452 |
+
# Create the message content
|
| 453 |
+
message = Message(content=content, role="assistant")
|
| 454 |
+
|
| 455 |
+
# Create choices
|
| 456 |
+
choice = Choices(finish_reason="stop", index=0, message=message)
|
| 457 |
+
|
| 458 |
+
# Update model response
|
| 459 |
+
model_response.choices = [choice]
|
| 460 |
+
model_response.model = usage_info.get("model", model)
|
| 461 |
+
|
| 462 |
+
# Add usage information if available
|
| 463 |
+
if usage_info:
|
| 464 |
+
from litellm.types.utils import Usage
|
| 465 |
+
|
| 466 |
+
usage = Usage(
|
| 467 |
+
prompt_tokens=usage_info.get("inputTokens", 0),
|
| 468 |
+
completion_tokens=usage_info.get("outputTokens", 0),
|
| 469 |
+
total_tokens=usage_info.get("inputTokens", 0)
|
| 470 |
+
+ usage_info.get("outputTokens", 0),
|
| 471 |
+
)
|
| 472 |
+
setattr(model_response, "usage", usage)
|
| 473 |
+
|
| 474 |
+
return model_response
|
| 475 |
+
|
| 476 |
+
def transform_response(
|
| 477 |
+
self,
|
| 478 |
+
model: str,
|
| 479 |
+
raw_response: httpx.Response,
|
| 480 |
+
model_response: ModelResponse,
|
| 481 |
+
logging_obj: LiteLLMLoggingObj,
|
| 482 |
+
request_data: dict,
|
| 483 |
+
messages: List[AllMessageValues],
|
| 484 |
+
optional_params: dict,
|
| 485 |
+
litellm_params: dict,
|
| 486 |
+
encoding: Any,
|
| 487 |
+
api_key: Optional[str] = None,
|
| 488 |
+
json_mode: Optional[bool] = None,
|
| 489 |
+
) -> ModelResponse:
|
| 490 |
+
try:
|
| 491 |
+
# Get the raw binary content
|
| 492 |
+
raw_content = raw_response.content
|
| 493 |
+
verbose_logger.debug(
|
| 494 |
+
f"Processing {len(raw_content)} bytes of AWS event stream data"
|
| 495 |
+
)
|
| 496 |
+
|
| 497 |
+
# Parse the AWS event stream format
|
| 498 |
+
events = self._parse_aws_event_stream(raw_content)
|
| 499 |
+
verbose_logger.debug(f"Parsed {len(events)} events from stream")
|
| 500 |
+
|
| 501 |
+
# Extract response content from chunk events
|
| 502 |
+
content = self._extract_response_content(events)
|
| 503 |
+
|
| 504 |
+
# Extract usage information from trace events
|
| 505 |
+
usage_info = self._extract_usage_info(events)
|
| 506 |
+
|
| 507 |
+
# Build and return the model response
|
| 508 |
+
return self._build_model_response(
|
| 509 |
+
content=content,
|
| 510 |
+
model=model,
|
| 511 |
+
usage_info=usage_info,
|
| 512 |
+
model_response=model_response,
|
| 513 |
+
)
|
| 514 |
+
|
| 515 |
+
except Exception as e:
|
| 516 |
+
verbose_logger.error(
|
| 517 |
+
f"Error processing Bedrock Invoke Agent response: {str(e)}"
|
| 518 |
+
)
|
| 519 |
+
raise BedrockError(
|
| 520 |
+
message=f"Error processing response: {str(e)}",
|
| 521 |
+
status_code=raw_response.status_code,
|
| 522 |
+
)
|
| 523 |
+
|
| 524 |
+
def validate_environment(
|
| 525 |
+
self,
|
| 526 |
+
headers: dict,
|
| 527 |
+
model: str,
|
| 528 |
+
messages: List[AllMessageValues],
|
| 529 |
+
optional_params: dict,
|
| 530 |
+
litellm_params: dict,
|
| 531 |
+
api_key: Optional[str] = None,
|
| 532 |
+
api_base: Optional[str] = None,
|
| 533 |
+
) -> dict:
|
| 534 |
+
return headers
|
| 535 |
+
|
| 536 |
+
def get_error_class(
|
| 537 |
+
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
| 538 |
+
) -> BaseLLMException:
|
| 539 |
+
return BedrockError(status_code=status_code, message=error_message)
|
| 540 |
+
|
| 541 |
+
def should_fake_stream(
|
| 542 |
+
self,
|
| 543 |
+
model: Optional[str],
|
| 544 |
+
stream: Optional[bool],
|
| 545 |
+
custom_llm_provider: Optional[str] = None,
|
| 546 |
+
) -> bool:
|
| 547 |
+
return True
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_handler.py
ADDED
|
@@ -0,0 +1,1893 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
TODO: DELETE FILE. Bedrock LLM is no longer used. Goto `litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py`
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import copy
|
| 6 |
+
import time
|
| 7 |
+
import types
|
| 8 |
+
from functools import partial
|
| 9 |
+
from typing import (
|
| 10 |
+
AsyncIterator,
|
| 11 |
+
Callable,
|
| 12 |
+
Iterator,
|
| 13 |
+
Optional,
|
| 14 |
+
Tuple,
|
| 15 |
+
cast,
|
| 16 |
+
get_args,
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
import httpx # type: ignore
|
| 20 |
+
|
| 21 |
+
import litellm
|
| 22 |
+
from litellm import verbose_logger
|
| 23 |
+
from litellm._uuid import uuid
|
| 24 |
+
from litellm.caching.caching import InMemoryCache
|
| 25 |
+
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
| 26 |
+
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
| 27 |
+
from litellm.litellm_core_utils.litellm_logging import Logging
|
| 28 |
+
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
| 29 |
+
from litellm.litellm_core_utils.prompt_templates.factory import (
|
| 30 |
+
cohere_message_pt,
|
| 31 |
+
construct_tool_use_system_prompt,
|
| 32 |
+
contains_tag,
|
| 33 |
+
custom_prompt,
|
| 34 |
+
extract_between_tags,
|
| 35 |
+
parse_xml_params,
|
| 36 |
+
prompt_factory,
|
| 37 |
+
)
|
| 38 |
+
from litellm.llms.anthropic.chat.handler import (
|
| 39 |
+
ModelResponseIterator as AnthropicModelResponseIterator,
|
| 40 |
+
)
|
| 41 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 42 |
+
AsyncHTTPHandler,
|
| 43 |
+
HTTPHandler,
|
| 44 |
+
_get_httpx_client,
|
| 45 |
+
get_async_httpx_client,
|
| 46 |
+
)
|
| 47 |
+
from litellm.types.llms.bedrock import *
|
| 48 |
+
from litellm.types.llms.openai import (
|
| 49 |
+
ChatCompletionRedactedThinkingBlock,
|
| 50 |
+
ChatCompletionThinkingBlock,
|
| 51 |
+
ChatCompletionToolCallChunk,
|
| 52 |
+
ChatCompletionToolCallFunctionChunk,
|
| 53 |
+
ChatCompletionUsageBlock,
|
| 54 |
+
)
|
| 55 |
+
from litellm.types.utils import (
|
| 56 |
+
ChatCompletionMessageToolCall,
|
| 57 |
+
Choices,
|
| 58 |
+
Delta,
|
| 59 |
+
)
|
| 60 |
+
from litellm.types.utils import GenericStreamingChunk as GChunk
|
| 61 |
+
from litellm.types.utils import (
|
| 62 |
+
ModelResponse,
|
| 63 |
+
ModelResponseStream,
|
| 64 |
+
StreamingChoices,
|
| 65 |
+
Usage,
|
| 66 |
+
)
|
| 67 |
+
from litellm.utils import CustomStreamWrapper, get_secret
|
| 68 |
+
|
| 69 |
+
from ..base_aws_llm import BaseAWSLLM
|
| 70 |
+
from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name
|
| 71 |
+
|
| 72 |
+
_response_stream_shape_cache = None
|
| 73 |
+
bedrock_tool_name_mappings: InMemoryCache = InMemoryCache(
|
| 74 |
+
max_size_in_memory=50, default_ttl=600
|
| 75 |
+
)
|
| 76 |
+
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
| 77 |
+
from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
|
| 78 |
+
AmazonBedrockOpenAIConfig,
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
converse_config = AmazonConverseConfig()
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class AmazonCohereChatConfig:
|
| 85 |
+
"""
|
| 86 |
+
Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command-r-plus.html
|
| 87 |
+
"""
|
| 88 |
+
|
| 89 |
+
documents: Optional[List[Document]] = None
|
| 90 |
+
search_queries_only: Optional[bool] = None
|
| 91 |
+
preamble: Optional[str] = None
|
| 92 |
+
max_tokens: Optional[int] = None
|
| 93 |
+
temperature: Optional[float] = None
|
| 94 |
+
p: Optional[float] = None
|
| 95 |
+
k: Optional[float] = None
|
| 96 |
+
prompt_truncation: Optional[str] = None
|
| 97 |
+
frequency_penalty: Optional[float] = None
|
| 98 |
+
presence_penalty: Optional[float] = None
|
| 99 |
+
seed: Optional[int] = None
|
| 100 |
+
return_prompt: Optional[bool] = None
|
| 101 |
+
stop_sequences: Optional[List[str]] = None
|
| 102 |
+
raw_prompting: Optional[bool] = None
|
| 103 |
+
|
| 104 |
+
def __init__(
|
| 105 |
+
self,
|
| 106 |
+
documents: Optional[List[Document]] = None,
|
| 107 |
+
search_queries_only: Optional[bool] = None,
|
| 108 |
+
preamble: Optional[str] = None,
|
| 109 |
+
max_tokens: Optional[int] = None,
|
| 110 |
+
temperature: Optional[float] = None,
|
| 111 |
+
p: Optional[float] = None,
|
| 112 |
+
k: Optional[float] = None,
|
| 113 |
+
prompt_truncation: Optional[str] = None,
|
| 114 |
+
frequency_penalty: Optional[float] = None,
|
| 115 |
+
presence_penalty: Optional[float] = None,
|
| 116 |
+
seed: Optional[int] = None,
|
| 117 |
+
return_prompt: Optional[bool] = None,
|
| 118 |
+
stop_sequences: Optional[str] = None,
|
| 119 |
+
raw_prompting: Optional[bool] = None,
|
| 120 |
+
) -> None:
|
| 121 |
+
locals_ = locals().copy()
|
| 122 |
+
for key, value in locals_.items():
|
| 123 |
+
if key != "self" and value is not None:
|
| 124 |
+
setattr(self.__class__, key, value)
|
| 125 |
+
|
| 126 |
+
@classmethod
|
| 127 |
+
def get_config(cls):
|
| 128 |
+
return {
|
| 129 |
+
k: v
|
| 130 |
+
for k, v in cls.__dict__.items()
|
| 131 |
+
if not k.startswith("__")
|
| 132 |
+
and not isinstance(
|
| 133 |
+
v,
|
| 134 |
+
(
|
| 135 |
+
types.FunctionType,
|
| 136 |
+
types.BuiltinFunctionType,
|
| 137 |
+
classmethod,
|
| 138 |
+
staticmethod,
|
| 139 |
+
),
|
| 140 |
+
)
|
| 141 |
+
and v is not None
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
def get_supported_openai_params(self) -> List[str]:
|
| 145 |
+
return [
|
| 146 |
+
"max_tokens",
|
| 147 |
+
"max_completion_tokens",
|
| 148 |
+
"stream",
|
| 149 |
+
"stop",
|
| 150 |
+
"temperature",
|
| 151 |
+
"top_p",
|
| 152 |
+
"frequency_penalty",
|
| 153 |
+
"presence_penalty",
|
| 154 |
+
"seed",
|
| 155 |
+
"stop",
|
| 156 |
+
"tools",
|
| 157 |
+
"tool_choice",
|
| 158 |
+
]
|
| 159 |
+
|
| 160 |
+
def map_openai_params(
|
| 161 |
+
self, non_default_params: dict, optional_params: dict
|
| 162 |
+
) -> dict:
|
| 163 |
+
for param, value in non_default_params.items():
|
| 164 |
+
if param == "max_tokens" or param == "max_completion_tokens":
|
| 165 |
+
optional_params["max_tokens"] = value
|
| 166 |
+
if param == "stream":
|
| 167 |
+
optional_params["stream"] = value
|
| 168 |
+
if param == "stop":
|
| 169 |
+
if isinstance(value, str):
|
| 170 |
+
value = [value]
|
| 171 |
+
optional_params["stop_sequences"] = value
|
| 172 |
+
if param == "temperature":
|
| 173 |
+
optional_params["temperature"] = value
|
| 174 |
+
if param == "top_p":
|
| 175 |
+
optional_params["p"] = value
|
| 176 |
+
if param == "frequency_penalty":
|
| 177 |
+
optional_params["frequency_penalty"] = value
|
| 178 |
+
if param == "presence_penalty":
|
| 179 |
+
optional_params["presence_penalty"] = value
|
| 180 |
+
if "seed":
|
| 181 |
+
optional_params["seed"] = value
|
| 182 |
+
return optional_params
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
async def make_call(
|
| 186 |
+
client: Optional[AsyncHTTPHandler],
|
| 187 |
+
api_base: str,
|
| 188 |
+
headers: dict,
|
| 189 |
+
data: str,
|
| 190 |
+
model: str,
|
| 191 |
+
messages: list,
|
| 192 |
+
logging_obj: Logging,
|
| 193 |
+
fake_stream: bool = False,
|
| 194 |
+
json_mode: Optional[bool] = False,
|
| 195 |
+
bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
|
| 196 |
+
stream_chunk_size: int = 1024,
|
| 197 |
+
):
|
| 198 |
+
try:
|
| 199 |
+
if client is None:
|
| 200 |
+
client = get_async_httpx_client(
|
| 201 |
+
llm_provider=litellm.LlmProviders.BEDROCK,
|
| 202 |
+
params={"ssl_verify": logging_obj.litellm_params.get("ssl_verify")}
|
| 203 |
+
if logging_obj
|
| 204 |
+
and logging_obj.litellm_params
|
| 205 |
+
and logging_obj.litellm_params.get("ssl_verify")
|
| 206 |
+
else None,
|
| 207 |
+
) # Create a new client if none provided
|
| 208 |
+
|
| 209 |
+
response = await client.post(
|
| 210 |
+
api_base,
|
| 211 |
+
headers=headers,
|
| 212 |
+
data=data,
|
| 213 |
+
stream=not fake_stream,
|
| 214 |
+
logging_obj=logging_obj,
|
| 215 |
+
)
|
| 216 |
+
|
| 217 |
+
if response.status_code != 200:
|
| 218 |
+
raise BedrockError(status_code=response.status_code, message=response.text)
|
| 219 |
+
|
| 220 |
+
if fake_stream:
|
| 221 |
+
model_response: (
|
| 222 |
+
ModelResponse
|
| 223 |
+
) = litellm.AmazonConverseConfig()._transform_response(
|
| 224 |
+
model=model,
|
| 225 |
+
response=response,
|
| 226 |
+
model_response=litellm.ModelResponse(),
|
| 227 |
+
stream=True,
|
| 228 |
+
logging_obj=logging_obj,
|
| 229 |
+
optional_params={},
|
| 230 |
+
api_key="",
|
| 231 |
+
data=data,
|
| 232 |
+
messages=messages,
|
| 233 |
+
encoding=litellm.encoding,
|
| 234 |
+
) # type: ignore
|
| 235 |
+
completion_stream: Any = MockResponseIterator(
|
| 236 |
+
model_response=model_response, json_mode=json_mode
|
| 237 |
+
)
|
| 238 |
+
elif bedrock_invoke_provider == "anthropic":
|
| 239 |
+
decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder(
|
| 240 |
+
model=model,
|
| 241 |
+
sync_stream=False,
|
| 242 |
+
json_mode=json_mode,
|
| 243 |
+
)
|
| 244 |
+
completion_stream = decoder.aiter_bytes(
|
| 245 |
+
response.aiter_bytes(chunk_size=stream_chunk_size)
|
| 246 |
+
)
|
| 247 |
+
elif bedrock_invoke_provider == "deepseek_r1":
|
| 248 |
+
decoder = AmazonDeepSeekR1StreamDecoder(
|
| 249 |
+
model=model,
|
| 250 |
+
sync_stream=False,
|
| 251 |
+
)
|
| 252 |
+
completion_stream = decoder.aiter_bytes(
|
| 253 |
+
response.aiter_bytes(chunk_size=stream_chunk_size)
|
| 254 |
+
)
|
| 255 |
+
else:
|
| 256 |
+
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
|
| 257 |
+
completion_stream = decoder.aiter_bytes(
|
| 258 |
+
response.aiter_bytes(chunk_size=stream_chunk_size)
|
| 259 |
+
)
|
| 260 |
+
|
| 261 |
+
# LOGGING
|
| 262 |
+
logging_obj.post_call(
|
| 263 |
+
input=messages,
|
| 264 |
+
api_key="",
|
| 265 |
+
original_response="first stream response received",
|
| 266 |
+
additional_args={"complete_input_dict": data},
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
return completion_stream
|
| 270 |
+
except httpx.HTTPStatusError as err:
|
| 271 |
+
error_code = err.response.status_code
|
| 272 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 273 |
+
except httpx.TimeoutException:
|
| 274 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 275 |
+
except Exception as e:
|
| 276 |
+
raise BedrockError(status_code=500, message=str(e))
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def make_sync_call(
|
| 280 |
+
client: Optional[HTTPHandler],
|
| 281 |
+
api_base: str,
|
| 282 |
+
headers: dict,
|
| 283 |
+
data: str,
|
| 284 |
+
signed_json_body: Optional[bytes],
|
| 285 |
+
model: str,
|
| 286 |
+
messages: list,
|
| 287 |
+
logging_obj: Logging,
|
| 288 |
+
fake_stream: bool = False,
|
| 289 |
+
json_mode: Optional[bool] = False,
|
| 290 |
+
bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
|
| 291 |
+
stream_chunk_size: int = 1024,
|
| 292 |
+
):
|
| 293 |
+
try:
|
| 294 |
+
if client is None:
|
| 295 |
+
client = _get_httpx_client(
|
| 296 |
+
params={"ssl_verify": logging_obj.litellm_params.get("ssl_verify")}
|
| 297 |
+
if logging_obj
|
| 298 |
+
and logging_obj.litellm_params
|
| 299 |
+
and logging_obj.litellm_params.get("ssl_verify")
|
| 300 |
+
else None
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
response = client.post(
|
| 304 |
+
api_base,
|
| 305 |
+
headers=headers,
|
| 306 |
+
data=signed_json_body if signed_json_body is not None else data,
|
| 307 |
+
stream=not fake_stream,
|
| 308 |
+
logging_obj=logging_obj,
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
if response.status_code != 200:
|
| 312 |
+
raise BedrockError(status_code=response.status_code, message=response.text)
|
| 313 |
+
|
| 314 |
+
if fake_stream:
|
| 315 |
+
model_response: (
|
| 316 |
+
ModelResponse
|
| 317 |
+
) = litellm.AmazonConverseConfig()._transform_response(
|
| 318 |
+
model=model,
|
| 319 |
+
response=response,
|
| 320 |
+
model_response=litellm.ModelResponse(),
|
| 321 |
+
stream=True,
|
| 322 |
+
logging_obj=logging_obj,
|
| 323 |
+
optional_params={},
|
| 324 |
+
api_key="",
|
| 325 |
+
data=data,
|
| 326 |
+
messages=messages,
|
| 327 |
+
encoding=litellm.encoding,
|
| 328 |
+
) # type: ignore
|
| 329 |
+
completion_stream: Any = MockResponseIterator(
|
| 330 |
+
model_response=model_response, json_mode=json_mode
|
| 331 |
+
)
|
| 332 |
+
elif bedrock_invoke_provider == "anthropic":
|
| 333 |
+
decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder(
|
| 334 |
+
model=model,
|
| 335 |
+
sync_stream=True,
|
| 336 |
+
json_mode=json_mode,
|
| 337 |
+
)
|
| 338 |
+
completion_stream = decoder.iter_bytes(
|
| 339 |
+
response.iter_bytes(chunk_size=stream_chunk_size)
|
| 340 |
+
)
|
| 341 |
+
elif bedrock_invoke_provider == "deepseek_r1":
|
| 342 |
+
decoder = AmazonDeepSeekR1StreamDecoder(
|
| 343 |
+
model=model,
|
| 344 |
+
sync_stream=True,
|
| 345 |
+
)
|
| 346 |
+
completion_stream = decoder.iter_bytes(
|
| 347 |
+
response.iter_bytes(chunk_size=stream_chunk_size)
|
| 348 |
+
)
|
| 349 |
+
else:
|
| 350 |
+
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
|
| 351 |
+
completion_stream = decoder.iter_bytes(
|
| 352 |
+
response.iter_bytes(chunk_size=stream_chunk_size)
|
| 353 |
+
)
|
| 354 |
+
|
| 355 |
+
# LOGGING
|
| 356 |
+
logging_obj.post_call(
|
| 357 |
+
input=messages,
|
| 358 |
+
api_key="",
|
| 359 |
+
original_response="first stream response received",
|
| 360 |
+
additional_args={"complete_input_dict": data},
|
| 361 |
+
)
|
| 362 |
+
|
| 363 |
+
return completion_stream
|
| 364 |
+
except httpx.HTTPStatusError as err:
|
| 365 |
+
error_code = err.response.status_code
|
| 366 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 367 |
+
except httpx.TimeoutException:
|
| 368 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 369 |
+
except Exception as e:
|
| 370 |
+
raise BedrockError(status_code=500, message=str(e))
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
class BedrockLLM(BaseAWSLLM):
|
| 374 |
+
"""
|
| 375 |
+
Example call
|
| 376 |
+
|
| 377 |
+
```
|
| 378 |
+
curl --location --request POST 'https://bedrock-runtime.{aws_region_name}.amazonaws.com/model/{bedrock_model_name}/invoke' \
|
| 379 |
+
--header 'Content-Type: application/json' \
|
| 380 |
+
--header 'Accept: application/json' \
|
| 381 |
+
--user "$AWS_ACCESS_KEY_ID":"$AWS_SECRET_ACCESS_KEY" \
|
| 382 |
+
--aws-sigv4 "aws:amz:us-east-1:bedrock" \
|
| 383 |
+
--data-raw '{
|
| 384 |
+
"prompt": "Hi",
|
| 385 |
+
"temperature": 0,
|
| 386 |
+
"p": 0.9,
|
| 387 |
+
"max_tokens": 4096
|
| 388 |
+
}'
|
| 389 |
+
```
|
| 390 |
+
"""
|
| 391 |
+
|
| 392 |
+
def __init__(self) -> None:
|
| 393 |
+
super().__init__()
|
| 394 |
+
|
| 395 |
+
@staticmethod
|
| 396 |
+
def is_claude_messages_api_model(model: str) -> bool:
|
| 397 |
+
"""
|
| 398 |
+
Check if the model uses the Claude Messages API (Claude 3+).
|
| 399 |
+
|
| 400 |
+
Handles:
|
| 401 |
+
- Regional prefixes: eu.anthropic.claude-*, us.anthropic.claude-*
|
| 402 |
+
- Claude 3 models: claude-3-haiku, claude-3-sonnet, claude-3-opus, claude-3-5-*, claude-3-7-*
|
| 403 |
+
- Claude 4 models: claude-opus-4, claude-sonnet-4, claude-haiku-4
|
| 404 |
+
"""
|
| 405 |
+
# Normalize model string to lowercase for matching
|
| 406 |
+
model_lower = model.lower()
|
| 407 |
+
|
| 408 |
+
# Claude 3+ indicators (all use Messages API)
|
| 409 |
+
messages_api_indicators = [
|
| 410 |
+
"claude-3", # Claude 3.x models
|
| 411 |
+
"claude-opus-4", # Claude Opus 4
|
| 412 |
+
"claude-sonnet-4", # Claude Sonnet 4
|
| 413 |
+
"claude-haiku-4", # Claude Haiku 4
|
| 414 |
+
]
|
| 415 |
+
|
| 416 |
+
return any(indicator in model_lower for indicator in messages_api_indicators)
|
| 417 |
+
|
| 418 |
+
def convert_messages_to_prompt(
|
| 419 |
+
self, model, messages, provider, custom_prompt_dict
|
| 420 |
+
) -> Tuple[str, Optional[list]]:
|
| 421 |
+
# handle anthropic prompts and amazon titan prompts
|
| 422 |
+
prompt = ""
|
| 423 |
+
chat_history: Optional[list] = None
|
| 424 |
+
## CUSTOM PROMPT
|
| 425 |
+
if model in custom_prompt_dict:
|
| 426 |
+
# check if the model has a registered custom prompt
|
| 427 |
+
model_prompt_details = custom_prompt_dict[model]
|
| 428 |
+
prompt = custom_prompt(
|
| 429 |
+
role_dict=model_prompt_details["roles"],
|
| 430 |
+
initial_prompt_value=model_prompt_details.get(
|
| 431 |
+
"initial_prompt_value", ""
|
| 432 |
+
),
|
| 433 |
+
final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
|
| 434 |
+
messages=messages,
|
| 435 |
+
)
|
| 436 |
+
return prompt, None
|
| 437 |
+
## ELSE
|
| 438 |
+
if provider == "anthropic" or provider == "amazon":
|
| 439 |
+
prompt = prompt_factory(
|
| 440 |
+
model=model, messages=messages, custom_llm_provider="bedrock"
|
| 441 |
+
)
|
| 442 |
+
elif provider == "mistral":
|
| 443 |
+
prompt = prompt_factory(
|
| 444 |
+
model=model, messages=messages, custom_llm_provider="bedrock"
|
| 445 |
+
)
|
| 446 |
+
elif provider == "meta" or provider == "llama":
|
| 447 |
+
prompt = prompt_factory(
|
| 448 |
+
model=model, messages=messages, custom_llm_provider="bedrock"
|
| 449 |
+
)
|
| 450 |
+
elif provider == "openai":
|
| 451 |
+
# OpenAI uses messages directly, no prompt conversion needed
|
| 452 |
+
# Return empty prompt as it won't be used
|
| 453 |
+
prompt = ""
|
| 454 |
+
elif provider == "cohere":
|
| 455 |
+
prompt, chat_history = cohere_message_pt(messages=messages)
|
| 456 |
+
else:
|
| 457 |
+
prompt = ""
|
| 458 |
+
for message in messages:
|
| 459 |
+
if "role" in message:
|
| 460 |
+
if message["role"] == "user":
|
| 461 |
+
prompt += f"{message['content']}"
|
| 462 |
+
else:
|
| 463 |
+
prompt += f"{message['content']}"
|
| 464 |
+
else:
|
| 465 |
+
prompt += f"{message['content']}"
|
| 466 |
+
return prompt, chat_history # type: ignore
|
| 467 |
+
|
| 468 |
+
def process_response( # noqa: PLR0915
|
| 469 |
+
self,
|
| 470 |
+
model: str,
|
| 471 |
+
response: httpx.Response,
|
| 472 |
+
model_response: ModelResponse,
|
| 473 |
+
stream: Optional[bool],
|
| 474 |
+
logging_obj: Logging,
|
| 475 |
+
optional_params: dict,
|
| 476 |
+
api_key: str,
|
| 477 |
+
data: Union[dict, str],
|
| 478 |
+
messages: List,
|
| 479 |
+
print_verbose,
|
| 480 |
+
encoding,
|
| 481 |
+
) -> Union[ModelResponse, CustomStreamWrapper]:
|
| 482 |
+
provider = self.get_bedrock_invoke_provider(model)
|
| 483 |
+
## LOGGING
|
| 484 |
+
logging_obj.post_call(
|
| 485 |
+
input=messages,
|
| 486 |
+
api_key=api_key,
|
| 487 |
+
original_response=response.text,
|
| 488 |
+
additional_args={"complete_input_dict": data},
|
| 489 |
+
)
|
| 490 |
+
print_verbose(f"raw model_response: {response.text}")
|
| 491 |
+
|
| 492 |
+
## RESPONSE OBJECT
|
| 493 |
+
try:
|
| 494 |
+
completion_response = response.json()
|
| 495 |
+
except Exception:
|
| 496 |
+
raise BedrockError(message=response.text, status_code=422)
|
| 497 |
+
|
| 498 |
+
outputText: Optional[str] = None
|
| 499 |
+
try:
|
| 500 |
+
if provider == "cohere":
|
| 501 |
+
if "text" in completion_response:
|
| 502 |
+
outputText = completion_response["text"] # type: ignore
|
| 503 |
+
elif "generations" in completion_response:
|
| 504 |
+
outputText = completion_response["generations"][0]["text"]
|
| 505 |
+
model_response.choices[0].finish_reason = map_finish_reason(
|
| 506 |
+
completion_response["generations"][0]["finish_reason"]
|
| 507 |
+
)
|
| 508 |
+
elif provider == "anthropic":
|
| 509 |
+
if self.is_claude_messages_api_model(model):
|
| 510 |
+
json_schemas: dict = {}
|
| 511 |
+
_is_function_call = False
|
| 512 |
+
## Handle Tool Calling
|
| 513 |
+
if "tools" in optional_params:
|
| 514 |
+
_is_function_call = True
|
| 515 |
+
for tool in optional_params["tools"]:
|
| 516 |
+
json_schemas[tool["function"]["name"]] = tool[
|
| 517 |
+
"function"
|
| 518 |
+
].get("parameters", None)
|
| 519 |
+
outputText = completion_response.get("content")[0].get("text", None)
|
| 520 |
+
if outputText is not None and contains_tag(
|
| 521 |
+
"invoke", outputText
|
| 522 |
+
): # OUTPUT PARSE FUNCTION CALL
|
| 523 |
+
function_name = extract_between_tags("tool_name", outputText)[0]
|
| 524 |
+
function_arguments_str = extract_between_tags(
|
| 525 |
+
"invoke", outputText
|
| 526 |
+
)[0].strip()
|
| 527 |
+
function_arguments_str = (
|
| 528 |
+
f"<invoke>{function_arguments_str}</invoke>"
|
| 529 |
+
)
|
| 530 |
+
function_arguments = parse_xml_params(
|
| 531 |
+
function_arguments_str,
|
| 532 |
+
json_schema=json_schemas.get(
|
| 533 |
+
function_name, None
|
| 534 |
+
), # check if we have a json schema for this function name)
|
| 535 |
+
)
|
| 536 |
+
_message = litellm.Message(
|
| 537 |
+
tool_calls=[
|
| 538 |
+
{
|
| 539 |
+
"id": f"call_{uuid.uuid4()}",
|
| 540 |
+
"type": "function",
|
| 541 |
+
"function": {
|
| 542 |
+
"name": function_name,
|
| 543 |
+
"arguments": json.dumps(function_arguments),
|
| 544 |
+
},
|
| 545 |
+
}
|
| 546 |
+
],
|
| 547 |
+
content=None,
|
| 548 |
+
)
|
| 549 |
+
model_response.choices[0].message = _message # type: ignore
|
| 550 |
+
model_response._hidden_params[
|
| 551 |
+
"original_response"
|
| 552 |
+
] = outputText # allow user to access raw anthropic tool calling response
|
| 553 |
+
if (
|
| 554 |
+
_is_function_call is True
|
| 555 |
+
and stream is not None
|
| 556 |
+
and stream is True
|
| 557 |
+
):
|
| 558 |
+
print_verbose(
|
| 559 |
+
"INSIDE BEDROCK STREAMING TOOL CALLING CONDITION BLOCK"
|
| 560 |
+
)
|
| 561 |
+
# return an iterator
|
| 562 |
+
streaming_model_response = ModelResponseStream()
|
| 563 |
+
streaming_model_response.choices[0].finish_reason = getattr(
|
| 564 |
+
model_response.choices[0], "finish_reason", "stop"
|
| 565 |
+
)
|
| 566 |
+
# streaming_model_response.choices = [litellm.utils.StreamingChoices()]
|
| 567 |
+
streaming_choice = litellm.utils.StreamingChoices()
|
| 568 |
+
streaming_choice.index = model_response.choices[0].index
|
| 569 |
+
_tool_calls = []
|
| 570 |
+
print_verbose(
|
| 571 |
+
f"type of model_response.choices[0]: {type(model_response.choices[0])}"
|
| 572 |
+
)
|
| 573 |
+
print_verbose(
|
| 574 |
+
f"type of streaming_choice: {type(streaming_choice)}"
|
| 575 |
+
)
|
| 576 |
+
if isinstance(model_response.choices[0], litellm.Choices):
|
| 577 |
+
if getattr(
|
| 578 |
+
model_response.choices[0].message, "tool_calls", None
|
| 579 |
+
) is not None and isinstance(
|
| 580 |
+
model_response.choices[0].message.tool_calls, list
|
| 581 |
+
):
|
| 582 |
+
for tool_call in model_response.choices[
|
| 583 |
+
0
|
| 584 |
+
].message.tool_calls:
|
| 585 |
+
_tool_call = {**tool_call.dict(), "index": 0}
|
| 586 |
+
_tool_calls.append(_tool_call)
|
| 587 |
+
delta_obj = Delta(
|
| 588 |
+
content=getattr(
|
| 589 |
+
model_response.choices[0].message, "content", None
|
| 590 |
+
),
|
| 591 |
+
role=model_response.choices[0].message.role,
|
| 592 |
+
tool_calls=_tool_calls,
|
| 593 |
+
)
|
| 594 |
+
streaming_choice.delta = delta_obj
|
| 595 |
+
streaming_model_response.choices = [streaming_choice]
|
| 596 |
+
completion_stream = ModelResponseIterator(
|
| 597 |
+
model_response=streaming_model_response
|
| 598 |
+
)
|
| 599 |
+
print_verbose(
|
| 600 |
+
"Returns anthropic CustomStreamWrapper with 'cached_response' streaming object"
|
| 601 |
+
)
|
| 602 |
+
return litellm.CustomStreamWrapper(
|
| 603 |
+
completion_stream=completion_stream,
|
| 604 |
+
model=model,
|
| 605 |
+
custom_llm_provider="cached_response",
|
| 606 |
+
logging_obj=logging_obj,
|
| 607 |
+
)
|
| 608 |
+
|
| 609 |
+
model_response.choices[0].finish_reason = map_finish_reason(
|
| 610 |
+
completion_response.get("stop_reason", "")
|
| 611 |
+
)
|
| 612 |
+
_usage = litellm.Usage(
|
| 613 |
+
prompt_tokens=completion_response["usage"]["input_tokens"],
|
| 614 |
+
completion_tokens=completion_response["usage"]["output_tokens"],
|
| 615 |
+
total_tokens=completion_response["usage"]["input_tokens"]
|
| 616 |
+
+ completion_response["usage"]["output_tokens"],
|
| 617 |
+
)
|
| 618 |
+
setattr(model_response, "usage", _usage)
|
| 619 |
+
else:
|
| 620 |
+
outputText = completion_response["completion"]
|
| 621 |
+
|
| 622 |
+
model_response.choices[0].finish_reason = completion_response[
|
| 623 |
+
"stop_reason"
|
| 624 |
+
]
|
| 625 |
+
elif provider == "ai21":
|
| 626 |
+
outputText = (
|
| 627 |
+
completion_response.get("completions")[0].get("data").get("text")
|
| 628 |
+
)
|
| 629 |
+
elif provider == "meta" or provider == "llama":
|
| 630 |
+
outputText = completion_response["generation"]
|
| 631 |
+
elif provider == "openai":
|
| 632 |
+
# OpenAI imported models use OpenAI Chat Completions format
|
| 633 |
+
if (
|
| 634 |
+
"choices" in completion_response
|
| 635 |
+
and len(completion_response["choices"]) > 0
|
| 636 |
+
):
|
| 637 |
+
choice = completion_response["choices"][0]
|
| 638 |
+
if "message" in choice:
|
| 639 |
+
outputText = choice["message"].get("content")
|
| 640 |
+
elif "text" in choice: # fallback for completion format
|
| 641 |
+
outputText = choice["text"]
|
| 642 |
+
|
| 643 |
+
# Set finish reason
|
| 644 |
+
if "finish_reason" in choice:
|
| 645 |
+
model_response.choices[0].finish_reason = map_finish_reason(
|
| 646 |
+
choice["finish_reason"]
|
| 647 |
+
)
|
| 648 |
+
|
| 649 |
+
# Set usage if available
|
| 650 |
+
if "usage" in completion_response:
|
| 651 |
+
usage = completion_response["usage"]
|
| 652 |
+
_usage = litellm.Usage(
|
| 653 |
+
prompt_tokens=usage.get("prompt_tokens", 0),
|
| 654 |
+
completion_tokens=usage.get("completion_tokens", 0),
|
| 655 |
+
total_tokens=usage.get("total_tokens", 0),
|
| 656 |
+
)
|
| 657 |
+
setattr(model_response, "usage", _usage)
|
| 658 |
+
elif provider == "mistral":
|
| 659 |
+
outputText = completion_response["outputs"][0]["text"]
|
| 660 |
+
model_response.choices[0].finish_reason = completion_response[
|
| 661 |
+
"outputs"
|
| 662 |
+
][0]["stop_reason"]
|
| 663 |
+
else: # amazon titan
|
| 664 |
+
outputText = completion_response.get("results")[0].get("outputText")
|
| 665 |
+
except Exception as e:
|
| 666 |
+
raise BedrockError(
|
| 667 |
+
message="Error processing={}, Received error={}".format(
|
| 668 |
+
response.text, str(e)
|
| 669 |
+
),
|
| 670 |
+
status_code=422,
|
| 671 |
+
)
|
| 672 |
+
|
| 673 |
+
try:
|
| 674 |
+
if (
|
| 675 |
+
outputText is not None
|
| 676 |
+
and len(outputText) > 0
|
| 677 |
+
and hasattr(model_response.choices[0], "message")
|
| 678 |
+
and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore
|
| 679 |
+
is None
|
| 680 |
+
):
|
| 681 |
+
model_response.choices[0].message.content = outputText # type: ignore
|
| 682 |
+
elif (
|
| 683 |
+
hasattr(model_response.choices[0], "message")
|
| 684 |
+
and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore
|
| 685 |
+
is not None
|
| 686 |
+
):
|
| 687 |
+
pass
|
| 688 |
+
else:
|
| 689 |
+
raise Exception()
|
| 690 |
+
except Exception as e:
|
| 691 |
+
raise BedrockError(
|
| 692 |
+
message="Error parsing received text={}.\nError-{}".format(
|
| 693 |
+
outputText, str(e)
|
| 694 |
+
),
|
| 695 |
+
status_code=response.status_code,
|
| 696 |
+
)
|
| 697 |
+
|
| 698 |
+
if stream and provider == "ai21":
|
| 699 |
+
streaming_model_response = ModelResponseStream()
|
| 700 |
+
streaming_model_response.choices[0].finish_reason = model_response.choices[ # type: ignore
|
| 701 |
+
0
|
| 702 |
+
].finish_reason
|
| 703 |
+
# streaming_model_response.choices = [litellm.utils.StreamingChoices()]
|
| 704 |
+
streaming_choice = litellm.utils.StreamingChoices()
|
| 705 |
+
streaming_choice.index = model_response.choices[0].index
|
| 706 |
+
delta_obj = litellm.utils.Delta(
|
| 707 |
+
content=getattr(model_response.choices[0].message, "content", None), # type: ignore
|
| 708 |
+
role=model_response.choices[0].message.role, # type: ignore
|
| 709 |
+
)
|
| 710 |
+
streaming_choice.delta = delta_obj
|
| 711 |
+
streaming_model_response.choices = [streaming_choice]
|
| 712 |
+
mri = ModelResponseIterator(model_response=streaming_model_response)
|
| 713 |
+
return CustomStreamWrapper(
|
| 714 |
+
completion_stream=mri,
|
| 715 |
+
model=model,
|
| 716 |
+
custom_llm_provider="cached_response",
|
| 717 |
+
logging_obj=logging_obj,
|
| 718 |
+
)
|
| 719 |
+
|
| 720 |
+
## CALCULATING USAGE - bedrock returns usage in the headers
|
| 721 |
+
# Skip if usage was already set (e.g., from JSON response for OpenAI provider)
|
| 722 |
+
if (
|
| 723 |
+
not hasattr(model_response, "usage")
|
| 724 |
+
or getattr(model_response, "usage", None) is None
|
| 725 |
+
):
|
| 726 |
+
bedrock_input_tokens = response.headers.get(
|
| 727 |
+
"x-amzn-bedrock-input-token-count", None
|
| 728 |
+
)
|
| 729 |
+
bedrock_output_tokens = response.headers.get(
|
| 730 |
+
"x-amzn-bedrock-output-token-count", None
|
| 731 |
+
)
|
| 732 |
+
|
| 733 |
+
prompt_tokens = int(
|
| 734 |
+
bedrock_input_tokens or litellm.token_counter(messages=messages)
|
| 735 |
+
)
|
| 736 |
+
|
| 737 |
+
completion_tokens = int(
|
| 738 |
+
bedrock_output_tokens
|
| 739 |
+
or litellm.token_counter(
|
| 740 |
+
text=model_response.choices[0].message.content, # type: ignore
|
| 741 |
+
count_response_tokens=True,
|
| 742 |
+
)
|
| 743 |
+
)
|
| 744 |
+
|
| 745 |
+
model_response.created = int(time.time())
|
| 746 |
+
model_response.model = model
|
| 747 |
+
usage = Usage(
|
| 748 |
+
prompt_tokens=prompt_tokens,
|
| 749 |
+
completion_tokens=completion_tokens,
|
| 750 |
+
total_tokens=prompt_tokens + completion_tokens,
|
| 751 |
+
)
|
| 752 |
+
setattr(model_response, "usage", usage)
|
| 753 |
+
else:
|
| 754 |
+
# Ensure created and model are set even if usage was already set
|
| 755 |
+
model_response.created = int(time.time())
|
| 756 |
+
model_response.model = model
|
| 757 |
+
|
| 758 |
+
return model_response
|
| 759 |
+
|
| 760 |
+
def completion( # noqa: PLR0915
|
| 761 |
+
self,
|
| 762 |
+
model: str,
|
| 763 |
+
messages: list,
|
| 764 |
+
api_base: Optional[str],
|
| 765 |
+
custom_prompt_dict: dict,
|
| 766 |
+
model_response: ModelResponse,
|
| 767 |
+
print_verbose: Callable,
|
| 768 |
+
encoding,
|
| 769 |
+
logging_obj: Logging,
|
| 770 |
+
optional_params: dict,
|
| 771 |
+
acompletion: bool,
|
| 772 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 773 |
+
litellm_params=None,
|
| 774 |
+
logger_fn=None,
|
| 775 |
+
extra_headers: Optional[dict] = None,
|
| 776 |
+
client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None,
|
| 777 |
+
) -> Union[ModelResponse, CustomStreamWrapper]:
|
| 778 |
+
try:
|
| 779 |
+
from botocore.credentials import Credentials
|
| 780 |
+
except ImportError:
|
| 781 |
+
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
|
| 782 |
+
|
| 783 |
+
## SETUP ##
|
| 784 |
+
stream = optional_params.pop("stream", None)
|
| 785 |
+
stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
|
| 786 |
+
|
| 787 |
+
provider = self.get_bedrock_invoke_provider(model)
|
| 788 |
+
modelId = self.get_bedrock_model_id(
|
| 789 |
+
model=model,
|
| 790 |
+
provider=provider,
|
| 791 |
+
optional_params=optional_params,
|
| 792 |
+
)
|
| 793 |
+
|
| 794 |
+
## CREDENTIALS ##
|
| 795 |
+
# pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
|
| 796 |
+
aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
|
| 797 |
+
aws_access_key_id = optional_params.pop("aws_access_key_id", None)
|
| 798 |
+
aws_session_token = optional_params.pop("aws_session_token", None)
|
| 799 |
+
aws_region_name = optional_params.pop("aws_region_name", None)
|
| 800 |
+
aws_role_name = optional_params.pop("aws_role_name", None)
|
| 801 |
+
aws_session_name = optional_params.pop("aws_session_name", None)
|
| 802 |
+
aws_profile_name = optional_params.pop("aws_profile_name", None)
|
| 803 |
+
aws_bedrock_runtime_endpoint = optional_params.pop(
|
| 804 |
+
"aws_bedrock_runtime_endpoint", None
|
| 805 |
+
) # https://bedrock-runtime.{region_name}.amazonaws.com
|
| 806 |
+
aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
|
| 807 |
+
aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
|
| 808 |
+
ssl_verify = optional_params.pop("ssl_verify", None)
|
| 809 |
+
|
| 810 |
+
### SET REGION NAME ###
|
| 811 |
+
if aws_region_name is None:
|
| 812 |
+
# check env #
|
| 813 |
+
litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
|
| 814 |
+
|
| 815 |
+
if litellm_aws_region_name is not None and isinstance(
|
| 816 |
+
litellm_aws_region_name, str
|
| 817 |
+
):
|
| 818 |
+
aws_region_name = litellm_aws_region_name
|
| 819 |
+
|
| 820 |
+
standard_aws_region_name = get_secret("AWS_REGION", None)
|
| 821 |
+
if standard_aws_region_name is not None and isinstance(
|
| 822 |
+
standard_aws_region_name, str
|
| 823 |
+
):
|
| 824 |
+
aws_region_name = standard_aws_region_name
|
| 825 |
+
|
| 826 |
+
if aws_region_name is None:
|
| 827 |
+
aws_region_name = "us-west-2"
|
| 828 |
+
|
| 829 |
+
credentials: Credentials = self.get_credentials(
|
| 830 |
+
aws_access_key_id=aws_access_key_id,
|
| 831 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 832 |
+
aws_session_token=aws_session_token,
|
| 833 |
+
aws_region_name=aws_region_name,
|
| 834 |
+
aws_session_name=aws_session_name,
|
| 835 |
+
aws_profile_name=aws_profile_name,
|
| 836 |
+
aws_role_name=aws_role_name,
|
| 837 |
+
aws_web_identity_token=aws_web_identity_token,
|
| 838 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 839 |
+
ssl_verify=ssl_verify,
|
| 840 |
+
)
|
| 841 |
+
|
| 842 |
+
### SET RUNTIME ENDPOINT ###
|
| 843 |
+
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
|
| 844 |
+
api_base=api_base,
|
| 845 |
+
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
| 846 |
+
aws_region_name=aws_region_name,
|
| 847 |
+
)
|
| 848 |
+
|
| 849 |
+
if (stream is not None and stream is True) and provider != "ai21":
|
| 850 |
+
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream"
|
| 851 |
+
proxy_endpoint_url = (
|
| 852 |
+
f"{proxy_endpoint_url}/model/{modelId}/invoke-with-response-stream"
|
| 853 |
+
)
|
| 854 |
+
else:
|
| 855 |
+
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
|
| 856 |
+
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
|
| 857 |
+
|
| 858 |
+
prompt, chat_history = self.convert_messages_to_prompt(
|
| 859 |
+
model, messages, provider, custom_prompt_dict
|
| 860 |
+
)
|
| 861 |
+
inference_params = copy.deepcopy(optional_params)
|
| 862 |
+
json_schemas: dict = {}
|
| 863 |
+
if provider == "cohere":
|
| 864 |
+
if model.startswith("cohere.command-r"):
|
| 865 |
+
## LOAD CONFIG
|
| 866 |
+
config = litellm.AmazonCohereChatConfig().get_config()
|
| 867 |
+
for k, v in config.items():
|
| 868 |
+
if (
|
| 869 |
+
k not in inference_params
|
| 870 |
+
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 871 |
+
inference_params[k] = v
|
| 872 |
+
_data = {"message": prompt, **inference_params}
|
| 873 |
+
if chat_history is not None:
|
| 874 |
+
_data["chat_history"] = chat_history
|
| 875 |
+
data = json.dumps(_data)
|
| 876 |
+
else:
|
| 877 |
+
## LOAD CONFIG
|
| 878 |
+
config = litellm.AmazonCohereConfig.get_config()
|
| 879 |
+
for k, v in config.items():
|
| 880 |
+
if (
|
| 881 |
+
k not in inference_params
|
| 882 |
+
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 883 |
+
inference_params[k] = v
|
| 884 |
+
if stream is True:
|
| 885 |
+
inference_params[
|
| 886 |
+
"stream"
|
| 887 |
+
] = True # cohere requires stream = True in inference params
|
| 888 |
+
data = json.dumps({"prompt": prompt, **inference_params})
|
| 889 |
+
elif provider == "anthropic":
|
| 890 |
+
if self.is_claude_messages_api_model(model):
|
| 891 |
+
# Separate system prompt from rest of message
|
| 892 |
+
system_prompt_idx: list[int] = []
|
| 893 |
+
system_messages: list[str] = []
|
| 894 |
+
for idx, message in enumerate(messages):
|
| 895 |
+
if message["role"] == "system":
|
| 896 |
+
system_messages.append(message["content"])
|
| 897 |
+
system_prompt_idx.append(idx)
|
| 898 |
+
if len(system_prompt_idx) > 0:
|
| 899 |
+
inference_params["system"] = "\n".join(system_messages)
|
| 900 |
+
messages = [
|
| 901 |
+
i for j, i in enumerate(messages) if j not in system_prompt_idx
|
| 902 |
+
]
|
| 903 |
+
# Format rest of message according to anthropic guidelines
|
| 904 |
+
messages = prompt_factory(
|
| 905 |
+
model=model, messages=messages, custom_llm_provider="anthropic_xml"
|
| 906 |
+
) # type: ignore
|
| 907 |
+
## LOAD CONFIG
|
| 908 |
+
config = litellm.AmazonAnthropicClaudeConfig.get_config()
|
| 909 |
+
for k, v in config.items():
|
| 910 |
+
if (
|
| 911 |
+
k not in inference_params
|
| 912 |
+
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 913 |
+
inference_params[k] = v
|
| 914 |
+
## Handle Tool Calling
|
| 915 |
+
if "tools" in inference_params:
|
| 916 |
+
_is_function_call = True
|
| 917 |
+
for tool in inference_params["tools"]:
|
| 918 |
+
json_schemas[tool["function"]["name"]] = tool["function"].get(
|
| 919 |
+
"parameters", None
|
| 920 |
+
)
|
| 921 |
+
tool_calling_system_prompt = construct_tool_use_system_prompt(
|
| 922 |
+
tools=inference_params["tools"]
|
| 923 |
+
)
|
| 924 |
+
inference_params["system"] = (
|
| 925 |
+
inference_params.get("system", "\n")
|
| 926 |
+
+ tool_calling_system_prompt
|
| 927 |
+
) # add the anthropic tool calling prompt to the system prompt
|
| 928 |
+
inference_params.pop("tools")
|
| 929 |
+
data = json.dumps({"messages": messages, **inference_params})
|
| 930 |
+
else:
|
| 931 |
+
## LOAD CONFIG
|
| 932 |
+
config = litellm.AmazonAnthropicConfig.get_config()
|
| 933 |
+
for k, v in config.items():
|
| 934 |
+
if (
|
| 935 |
+
k not in inference_params
|
| 936 |
+
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 937 |
+
inference_params[k] = v
|
| 938 |
+
data = json.dumps({"prompt": prompt, **inference_params})
|
| 939 |
+
elif provider == "ai21":
|
| 940 |
+
## LOAD CONFIG
|
| 941 |
+
config = litellm.AmazonAI21Config.get_config()
|
| 942 |
+
for k, v in config.items():
|
| 943 |
+
if (
|
| 944 |
+
k not in inference_params
|
| 945 |
+
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 946 |
+
inference_params[k] = v
|
| 947 |
+
|
| 948 |
+
data = json.dumps({"prompt": prompt, **inference_params})
|
| 949 |
+
elif provider == "mistral":
|
| 950 |
+
## LOAD CONFIG
|
| 951 |
+
config = litellm.AmazonMistralConfig.get_config()
|
| 952 |
+
for k, v in config.items():
|
| 953 |
+
if (
|
| 954 |
+
k not in inference_params
|
| 955 |
+
): # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 956 |
+
inference_params[k] = v
|
| 957 |
+
|
| 958 |
+
data = json.dumps({"prompt": prompt, **inference_params})
|
| 959 |
+
elif provider == "amazon": # amazon titan
|
| 960 |
+
## LOAD CONFIG
|
| 961 |
+
config = litellm.AmazonTitanConfig.get_config()
|
| 962 |
+
for k, v in config.items():
|
| 963 |
+
if (
|
| 964 |
+
k not in inference_params
|
| 965 |
+
): # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 966 |
+
inference_params[k] = v
|
| 967 |
+
|
| 968 |
+
data = json.dumps(
|
| 969 |
+
{
|
| 970 |
+
"inputText": prompt,
|
| 971 |
+
"textGenerationConfig": inference_params,
|
| 972 |
+
}
|
| 973 |
+
)
|
| 974 |
+
elif provider == "meta" or provider == "llama":
|
| 975 |
+
## LOAD CONFIG
|
| 976 |
+
config = litellm.AmazonLlamaConfig.get_config()
|
| 977 |
+
for k, v in config.items():
|
| 978 |
+
if (
|
| 979 |
+
k not in inference_params
|
| 980 |
+
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 981 |
+
inference_params[k] = v
|
| 982 |
+
data = json.dumps({"prompt": prompt, **inference_params})
|
| 983 |
+
elif provider == "openai":
|
| 984 |
+
## OpenAI imported models use OpenAI Chat Completions format (messages-based)
|
| 985 |
+
# Use AmazonBedrockOpenAIConfig for proper OpenAI transformation
|
| 986 |
+
openai_config = AmazonBedrockOpenAIConfig()
|
| 987 |
+
supported_params = openai_config.get_supported_openai_params(model=model)
|
| 988 |
+
|
| 989 |
+
# Filter to only supported OpenAI params
|
| 990 |
+
filtered_params = {
|
| 991 |
+
k: v for k, v in inference_params.items() if k in supported_params
|
| 992 |
+
}
|
| 993 |
+
|
| 994 |
+
# OpenAI uses messages format, not prompt
|
| 995 |
+
data = json.dumps({"messages": messages, **filtered_params})
|
| 996 |
+
else:
|
| 997 |
+
## LOGGING
|
| 998 |
+
logging_obj.pre_call(
|
| 999 |
+
input=messages,
|
| 1000 |
+
api_key="",
|
| 1001 |
+
additional_args={
|
| 1002 |
+
"complete_input_dict": inference_params,
|
| 1003 |
+
},
|
| 1004 |
+
)
|
| 1005 |
+
raise BedrockError(
|
| 1006 |
+
status_code=404,
|
| 1007 |
+
message="Bedrock Invoke HTTPX: Unknown provider={}, model={}. Try calling via converse route - `bedrock/converse/<model>`.".format(
|
| 1008 |
+
provider, model
|
| 1009 |
+
),
|
| 1010 |
+
)
|
| 1011 |
+
|
| 1012 |
+
## COMPLETION CALL
|
| 1013 |
+
|
| 1014 |
+
headers = {"Content-Type": "application/json"}
|
| 1015 |
+
if extra_headers is not None:
|
| 1016 |
+
headers = {"Content-Type": "application/json", **extra_headers}
|
| 1017 |
+
prepped = self.get_request_headers(
|
| 1018 |
+
credentials=credentials,
|
| 1019 |
+
aws_region_name=aws_region_name,
|
| 1020 |
+
extra_headers=extra_headers,
|
| 1021 |
+
endpoint_url=endpoint_url,
|
| 1022 |
+
data=data,
|
| 1023 |
+
headers=headers,
|
| 1024 |
+
)
|
| 1025 |
+
|
| 1026 |
+
## LOGGING
|
| 1027 |
+
logging_obj.pre_call(
|
| 1028 |
+
input=messages,
|
| 1029 |
+
api_key="",
|
| 1030 |
+
additional_args={
|
| 1031 |
+
"complete_input_dict": data,
|
| 1032 |
+
"api_base": proxy_endpoint_url,
|
| 1033 |
+
"headers": prepped.headers,
|
| 1034 |
+
},
|
| 1035 |
+
)
|
| 1036 |
+
|
| 1037 |
+
### ROUTING (ASYNC, STREAMING, SYNC)
|
| 1038 |
+
if acompletion:
|
| 1039 |
+
if isinstance(client, HTTPHandler):
|
| 1040 |
+
client = None
|
| 1041 |
+
if stream is True and provider != "ai21":
|
| 1042 |
+
return self.async_streaming(
|
| 1043 |
+
model=model,
|
| 1044 |
+
messages=messages,
|
| 1045 |
+
data=data,
|
| 1046 |
+
api_base=proxy_endpoint_url,
|
| 1047 |
+
model_response=model_response,
|
| 1048 |
+
print_verbose=print_verbose,
|
| 1049 |
+
encoding=encoding,
|
| 1050 |
+
logging_obj=logging_obj,
|
| 1051 |
+
optional_params=optional_params,
|
| 1052 |
+
stream=True,
|
| 1053 |
+
litellm_params=litellm_params,
|
| 1054 |
+
logger_fn=logger_fn,
|
| 1055 |
+
headers=prepped.headers,
|
| 1056 |
+
timeout=timeout,
|
| 1057 |
+
client=client,
|
| 1058 |
+
stream_chunk_size=stream_chunk_size,
|
| 1059 |
+
) # type: ignore
|
| 1060 |
+
### ASYNC COMPLETION
|
| 1061 |
+
return self.async_completion(
|
| 1062 |
+
model=model,
|
| 1063 |
+
messages=messages,
|
| 1064 |
+
data=data,
|
| 1065 |
+
api_base=proxy_endpoint_url,
|
| 1066 |
+
model_response=model_response,
|
| 1067 |
+
print_verbose=print_verbose,
|
| 1068 |
+
encoding=encoding,
|
| 1069 |
+
logging_obj=logging_obj,
|
| 1070 |
+
optional_params=optional_params,
|
| 1071 |
+
stream=stream, # type: ignore
|
| 1072 |
+
litellm_params=litellm_params,
|
| 1073 |
+
logger_fn=logger_fn,
|
| 1074 |
+
headers=prepped.headers,
|
| 1075 |
+
timeout=timeout,
|
| 1076 |
+
client=client,
|
| 1077 |
+
) # type: ignore
|
| 1078 |
+
|
| 1079 |
+
if client is None or isinstance(client, AsyncHTTPHandler):
|
| 1080 |
+
_params = {}
|
| 1081 |
+
if timeout is not None:
|
| 1082 |
+
if isinstance(timeout, float) or isinstance(timeout, int):
|
| 1083 |
+
timeout = httpx.Timeout(timeout)
|
| 1084 |
+
_params["timeout"] = timeout
|
| 1085 |
+
self.client = _get_httpx_client(_params) # type: ignore
|
| 1086 |
+
else:
|
| 1087 |
+
self.client = client
|
| 1088 |
+
if (stream is not None and stream is True) and provider != "ai21":
|
| 1089 |
+
response = self.client.post(
|
| 1090 |
+
url=proxy_endpoint_url,
|
| 1091 |
+
headers=prepped.headers, # type: ignore
|
| 1092 |
+
data=data,
|
| 1093 |
+
stream=stream,
|
| 1094 |
+
logging_obj=logging_obj,
|
| 1095 |
+
)
|
| 1096 |
+
|
| 1097 |
+
if response.status_code != 200:
|
| 1098 |
+
raise BedrockError(
|
| 1099 |
+
status_code=response.status_code, message=str(response.read())
|
| 1100 |
+
)
|
| 1101 |
+
|
| 1102 |
+
decoder = AWSEventStreamDecoder(model=model)
|
| 1103 |
+
|
| 1104 |
+
completion_stream = decoder.iter_bytes(
|
| 1105 |
+
response.iter_bytes(chunk_size=stream_chunk_size)
|
| 1106 |
+
)
|
| 1107 |
+
streaming_response = CustomStreamWrapper(
|
| 1108 |
+
completion_stream=completion_stream,
|
| 1109 |
+
model=model,
|
| 1110 |
+
custom_llm_provider="bedrock",
|
| 1111 |
+
logging_obj=logging_obj,
|
| 1112 |
+
)
|
| 1113 |
+
|
| 1114 |
+
## LOGGING
|
| 1115 |
+
logging_obj.post_call(
|
| 1116 |
+
input=messages,
|
| 1117 |
+
api_key="",
|
| 1118 |
+
original_response=streaming_response,
|
| 1119 |
+
additional_args={"complete_input_dict": data},
|
| 1120 |
+
)
|
| 1121 |
+
return streaming_response
|
| 1122 |
+
|
| 1123 |
+
try:
|
| 1124 |
+
response = self.client.post(
|
| 1125 |
+
url=proxy_endpoint_url,
|
| 1126 |
+
headers=dict(prepped.headers),
|
| 1127 |
+
data=data,
|
| 1128 |
+
logging_obj=logging_obj,
|
| 1129 |
+
)
|
| 1130 |
+
response.raise_for_status()
|
| 1131 |
+
except httpx.HTTPStatusError as err:
|
| 1132 |
+
error_code = err.response.status_code
|
| 1133 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 1134 |
+
except httpx.TimeoutException:
|
| 1135 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 1136 |
+
|
| 1137 |
+
return self.process_response(
|
| 1138 |
+
model=model,
|
| 1139 |
+
response=response,
|
| 1140 |
+
model_response=model_response,
|
| 1141 |
+
stream=stream,
|
| 1142 |
+
logging_obj=logging_obj,
|
| 1143 |
+
optional_params=optional_params,
|
| 1144 |
+
api_key="",
|
| 1145 |
+
data=data,
|
| 1146 |
+
messages=messages,
|
| 1147 |
+
print_verbose=print_verbose,
|
| 1148 |
+
encoding=encoding,
|
| 1149 |
+
)
|
| 1150 |
+
|
| 1151 |
+
async def async_completion(
|
| 1152 |
+
self,
|
| 1153 |
+
model: str,
|
| 1154 |
+
messages: list,
|
| 1155 |
+
api_base: str,
|
| 1156 |
+
model_response: ModelResponse,
|
| 1157 |
+
print_verbose: Callable,
|
| 1158 |
+
data: str,
|
| 1159 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 1160 |
+
encoding,
|
| 1161 |
+
logging_obj: Logging,
|
| 1162 |
+
stream,
|
| 1163 |
+
optional_params: dict,
|
| 1164 |
+
litellm_params=None,
|
| 1165 |
+
logger_fn=None,
|
| 1166 |
+
headers={},
|
| 1167 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 1168 |
+
) -> Union[ModelResponse, CustomStreamWrapper]:
|
| 1169 |
+
if client is None:
|
| 1170 |
+
_params = {}
|
| 1171 |
+
if timeout is not None:
|
| 1172 |
+
if isinstance(timeout, float) or isinstance(timeout, int):
|
| 1173 |
+
timeout = httpx.Timeout(timeout)
|
| 1174 |
+
_params["timeout"] = timeout
|
| 1175 |
+
client = get_async_httpx_client(params=_params, llm_provider=litellm.LlmProviders.BEDROCK) # type: ignore
|
| 1176 |
+
else:
|
| 1177 |
+
client = client # type: ignore
|
| 1178 |
+
|
| 1179 |
+
try:
|
| 1180 |
+
response = await client.post(
|
| 1181 |
+
api_base,
|
| 1182 |
+
headers=headers,
|
| 1183 |
+
data=data,
|
| 1184 |
+
timeout=timeout,
|
| 1185 |
+
logging_obj=logging_obj,
|
| 1186 |
+
)
|
| 1187 |
+
response.raise_for_status()
|
| 1188 |
+
except httpx.HTTPStatusError as err:
|
| 1189 |
+
error_code = err.response.status_code
|
| 1190 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 1191 |
+
except httpx.TimeoutException:
|
| 1192 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 1193 |
+
|
| 1194 |
+
return self.process_response(
|
| 1195 |
+
model=model,
|
| 1196 |
+
response=response,
|
| 1197 |
+
model_response=model_response,
|
| 1198 |
+
stream=stream if isinstance(stream, bool) else False,
|
| 1199 |
+
logging_obj=logging_obj,
|
| 1200 |
+
api_key="",
|
| 1201 |
+
data=data,
|
| 1202 |
+
messages=messages,
|
| 1203 |
+
print_verbose=print_verbose,
|
| 1204 |
+
optional_params=optional_params,
|
| 1205 |
+
encoding=encoding,
|
| 1206 |
+
)
|
| 1207 |
+
|
| 1208 |
+
@track_llm_api_timing() # for streaming, we need to instrument the function calling the wrapper
|
| 1209 |
+
async def async_streaming(
|
| 1210 |
+
self,
|
| 1211 |
+
model: str,
|
| 1212 |
+
messages: list,
|
| 1213 |
+
api_base: str,
|
| 1214 |
+
model_response: ModelResponse,
|
| 1215 |
+
print_verbose: Callable,
|
| 1216 |
+
data: str,
|
| 1217 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 1218 |
+
encoding,
|
| 1219 |
+
logging_obj: Logging,
|
| 1220 |
+
stream,
|
| 1221 |
+
optional_params: dict,
|
| 1222 |
+
litellm_params=None,
|
| 1223 |
+
logger_fn=None,
|
| 1224 |
+
headers={},
|
| 1225 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 1226 |
+
stream_chunk_size: int = 1024,
|
| 1227 |
+
) -> CustomStreamWrapper:
|
| 1228 |
+
# The call is not made here; instead, we prepare the necessary objects for the stream.
|
| 1229 |
+
|
| 1230 |
+
streaming_response = CustomStreamWrapper(
|
| 1231 |
+
completion_stream=None,
|
| 1232 |
+
make_call=partial(
|
| 1233 |
+
make_call,
|
| 1234 |
+
client=client,
|
| 1235 |
+
api_base=api_base,
|
| 1236 |
+
headers=headers,
|
| 1237 |
+
data=data, # type: ignore
|
| 1238 |
+
model=model,
|
| 1239 |
+
messages=messages,
|
| 1240 |
+
logging_obj=logging_obj,
|
| 1241 |
+
fake_stream=True if "ai21" in api_base else False,
|
| 1242 |
+
stream_chunk_size=stream_chunk_size,
|
| 1243 |
+
),
|
| 1244 |
+
model=model,
|
| 1245 |
+
custom_llm_provider="bedrock",
|
| 1246 |
+
logging_obj=logging_obj,
|
| 1247 |
+
)
|
| 1248 |
+
return streaming_response
|
| 1249 |
+
|
| 1250 |
+
@staticmethod
|
| 1251 |
+
def _get_provider_from_model_path(
|
| 1252 |
+
model_path: str,
|
| 1253 |
+
) -> Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL]:
|
| 1254 |
+
"""
|
| 1255 |
+
Helper function to get the provider from a model path with format: provider/model-name
|
| 1256 |
+
|
| 1257 |
+
Args:
|
| 1258 |
+
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
|
| 1259 |
+
|
| 1260 |
+
Returns:
|
| 1261 |
+
Optional[str]: The provider name, or None if no valid provider found
|
| 1262 |
+
"""
|
| 1263 |
+
parts = model_path.split("/")
|
| 1264 |
+
if len(parts) >= 1:
|
| 1265 |
+
provider = parts[0]
|
| 1266 |
+
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 1267 |
+
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
|
| 1268 |
+
return None
|
| 1269 |
+
|
| 1270 |
+
|
| 1271 |
+
def get_response_stream_shape():
|
| 1272 |
+
global _response_stream_shape_cache
|
| 1273 |
+
if _response_stream_shape_cache is None:
|
| 1274 |
+
from botocore.loaders import Loader
|
| 1275 |
+
from botocore.model import ServiceModel
|
| 1276 |
+
|
| 1277 |
+
loader = Loader()
|
| 1278 |
+
bedrock_service_dict = loader.load_service_model("bedrock-runtime", "service-2")
|
| 1279 |
+
bedrock_service_model = ServiceModel(bedrock_service_dict)
|
| 1280 |
+
_response_stream_shape_cache = bedrock_service_model.shape_for("ResponseStream")
|
| 1281 |
+
|
| 1282 |
+
return _response_stream_shape_cache
|
| 1283 |
+
|
| 1284 |
+
|
| 1285 |
+
class AWSEventStreamDecoder:
|
| 1286 |
+
def __init__(self, model: str, json_mode: Optional[bool] = False) -> None:
|
| 1287 |
+
from botocore.parsers import EventStreamJSONParser
|
| 1288 |
+
|
| 1289 |
+
self.model = model
|
| 1290 |
+
self.parser = EventStreamJSONParser()
|
| 1291 |
+
self.content_blocks: List[ContentBlockDeltaEvent] = []
|
| 1292 |
+
self.tool_calls_index: Optional[int] = None
|
| 1293 |
+
self.response_id: Optional[str] = None
|
| 1294 |
+
self.json_mode = json_mode
|
| 1295 |
+
self._current_tool_name: Optional[str] = None
|
| 1296 |
+
|
| 1297 |
+
def check_empty_tool_call_args(self) -> bool:
|
| 1298 |
+
"""
|
| 1299 |
+
Check if the tool call block so far has been an empty string
|
| 1300 |
+
"""
|
| 1301 |
+
args = ""
|
| 1302 |
+
# if text content block -> skip
|
| 1303 |
+
if len(self.content_blocks) == 0:
|
| 1304 |
+
return False
|
| 1305 |
+
|
| 1306 |
+
if (
|
| 1307 |
+
"toolUse" not in self.content_blocks[0]
|
| 1308 |
+
): # be explicit - only do this if tool use block, as this is to prevent json decoding errors
|
| 1309 |
+
return False
|
| 1310 |
+
|
| 1311 |
+
for block in self.content_blocks:
|
| 1312 |
+
if "toolUse" in block:
|
| 1313 |
+
args += block["toolUse"]["input"]
|
| 1314 |
+
|
| 1315 |
+
if len(args) == 0:
|
| 1316 |
+
return True
|
| 1317 |
+
return False
|
| 1318 |
+
|
| 1319 |
+
def extract_reasoning_content_str(
|
| 1320 |
+
self, reasoning_content_block: BedrockConverseReasoningContentBlockDelta
|
| 1321 |
+
) -> Optional[str]:
|
| 1322 |
+
if "text" in reasoning_content_block:
|
| 1323 |
+
return reasoning_content_block["text"]
|
| 1324 |
+
return None
|
| 1325 |
+
|
| 1326 |
+
def translate_thinking_blocks(
|
| 1327 |
+
self, thinking_block: BedrockConverseReasoningContentBlockDelta
|
| 1328 |
+
) -> Optional[
|
| 1329 |
+
List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]
|
| 1330 |
+
]:
|
| 1331 |
+
"""
|
| 1332 |
+
Translate the thinking blocks to a string
|
| 1333 |
+
"""
|
| 1334 |
+
|
| 1335 |
+
thinking_blocks_list: List[
|
| 1336 |
+
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
| 1337 |
+
] = []
|
| 1338 |
+
_thinking_block: Optional[
|
| 1339 |
+
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
| 1340 |
+
] = None
|
| 1341 |
+
|
| 1342 |
+
if "text" in thinking_block:
|
| 1343 |
+
_thinking_block = ChatCompletionThinkingBlock(type="thinking")
|
| 1344 |
+
_thinking_block["thinking"] = thinking_block["text"]
|
| 1345 |
+
elif "signature" in thinking_block:
|
| 1346 |
+
_thinking_block = ChatCompletionThinkingBlock(type="thinking")
|
| 1347 |
+
_thinking_block["signature"] = thinking_block["signature"]
|
| 1348 |
+
_thinking_block["thinking"] = "" # consistent with anthropic response
|
| 1349 |
+
elif "redactedContent" in thinking_block:
|
| 1350 |
+
_thinking_block = ChatCompletionRedactedThinkingBlock(
|
| 1351 |
+
type="redacted_thinking", data=thinking_block["redactedContent"]
|
| 1352 |
+
)
|
| 1353 |
+
if _thinking_block is not None:
|
| 1354 |
+
thinking_blocks_list.append(_thinking_block)
|
| 1355 |
+
return thinking_blocks_list
|
| 1356 |
+
|
| 1357 |
+
def _initialize_converse_response_id(self, chunk_data: dict):
|
| 1358 |
+
"""Initialize response_id from chunk data if not already set."""
|
| 1359 |
+
if self.response_id is None:
|
| 1360 |
+
if "messageStart" in chunk_data:
|
| 1361 |
+
conversation_id = chunk_data["messageStart"].get("conversationId")
|
| 1362 |
+
if conversation_id:
|
| 1363 |
+
self.response_id = f"chatcmpl-{conversation_id}"
|
| 1364 |
+
else:
|
| 1365 |
+
# Fallback to generating a UUID if the first chunk is not messageStart
|
| 1366 |
+
self.response_id = f"chatcmpl-{uuid.uuid4()}"
|
| 1367 |
+
|
| 1368 |
+
def _handle_converse_start_event(
|
| 1369 |
+
self,
|
| 1370 |
+
start_obj: ContentBlockStartEvent,
|
| 1371 |
+
) -> Tuple[
|
| 1372 |
+
Optional[ChatCompletionToolCallChunk],
|
| 1373 |
+
dict,
|
| 1374 |
+
Optional[
|
| 1375 |
+
List[
|
| 1376 |
+
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
| 1377 |
+
]
|
| 1378 |
+
],
|
| 1379 |
+
]:
|
| 1380 |
+
"""Handle 'start' event in converse chunk parsing."""
|
| 1381 |
+
tool_use: Optional[ChatCompletionToolCallChunk] = None
|
| 1382 |
+
provider_specific_fields: dict = {}
|
| 1383 |
+
thinking_blocks: Optional[
|
| 1384 |
+
List[
|
| 1385 |
+
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
| 1386 |
+
]
|
| 1387 |
+
] = None
|
| 1388 |
+
|
| 1389 |
+
self.content_blocks = [] # reset
|
| 1390 |
+
if start_obj is not None:
|
| 1391 |
+
if "toolUse" in start_obj and start_obj["toolUse"] is not None:
|
| 1392 |
+
## check tool name was formatted by litellm
|
| 1393 |
+
_response_tool_name = start_obj["toolUse"]["name"]
|
| 1394 |
+
response_tool_name = get_bedrock_tool_name(
|
| 1395 |
+
response_tool_name=_response_tool_name
|
| 1396 |
+
)
|
| 1397 |
+
self._current_tool_name = response_tool_name
|
| 1398 |
+
|
| 1399 |
+
# When json_mode is True, suppress the internal json_tool_call
|
| 1400 |
+
# and convert its content to text in delta events instead
|
| 1401 |
+
if (
|
| 1402 |
+
self.json_mode is True
|
| 1403 |
+
and response_tool_name == RESPONSE_FORMAT_TOOL_NAME
|
| 1404 |
+
):
|
| 1405 |
+
return tool_use, provider_specific_fields, thinking_blocks
|
| 1406 |
+
|
| 1407 |
+
self.tool_calls_index = (
|
| 1408 |
+
0 if self.tool_calls_index is None else self.tool_calls_index + 1
|
| 1409 |
+
)
|
| 1410 |
+
tool_use = {
|
| 1411 |
+
"id": start_obj["toolUse"]["toolUseId"],
|
| 1412 |
+
"type": "function",
|
| 1413 |
+
"function": {
|
| 1414 |
+
"name": response_tool_name,
|
| 1415 |
+
"arguments": "",
|
| 1416 |
+
},
|
| 1417 |
+
"index": self.tool_calls_index,
|
| 1418 |
+
}
|
| 1419 |
+
elif (
|
| 1420 |
+
"reasoningContent" in start_obj
|
| 1421 |
+
and start_obj["reasoningContent"] is not None
|
| 1422 |
+
): # redacted thinking can be in start object
|
| 1423 |
+
thinking_blocks = self.translate_thinking_blocks(
|
| 1424 |
+
start_obj["reasoningContent"]
|
| 1425 |
+
)
|
| 1426 |
+
provider_specific_fields = {
|
| 1427 |
+
"reasoningContent": start_obj["reasoningContent"],
|
| 1428 |
+
}
|
| 1429 |
+
return tool_use, provider_specific_fields, thinking_blocks
|
| 1430 |
+
|
| 1431 |
+
def _handle_converse_delta_event(
|
| 1432 |
+
self,
|
| 1433 |
+
delta_obj: ContentBlockDeltaEvent,
|
| 1434 |
+
index: int,
|
| 1435 |
+
) -> Tuple[
|
| 1436 |
+
str,
|
| 1437 |
+
Optional[ChatCompletionToolCallChunk],
|
| 1438 |
+
dict,
|
| 1439 |
+
Optional[str],
|
| 1440 |
+
Optional[
|
| 1441 |
+
List[
|
| 1442 |
+
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
| 1443 |
+
]
|
| 1444 |
+
],
|
| 1445 |
+
]:
|
| 1446 |
+
"""Handle 'delta' event in converse chunk parsing."""
|
| 1447 |
+
text = ""
|
| 1448 |
+
tool_use: Optional[ChatCompletionToolCallChunk] = None
|
| 1449 |
+
provider_specific_fields: dict = {}
|
| 1450 |
+
reasoning_content: Optional[str] = None
|
| 1451 |
+
thinking_blocks: Optional[
|
| 1452 |
+
List[
|
| 1453 |
+
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
|
| 1454 |
+
]
|
| 1455 |
+
] = None
|
| 1456 |
+
|
| 1457 |
+
self.content_blocks.append(delta_obj)
|
| 1458 |
+
if "text" in delta_obj:
|
| 1459 |
+
text = delta_obj["text"]
|
| 1460 |
+
elif "toolUse" in delta_obj:
|
| 1461 |
+
# When json_mode is True and this is the internal json_tool_call,
|
| 1462 |
+
# convert tool input to text content instead of tool call arguments
|
| 1463 |
+
if (
|
| 1464 |
+
self.json_mode is True
|
| 1465 |
+
and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME
|
| 1466 |
+
):
|
| 1467 |
+
text = delta_obj["toolUse"]["input"]
|
| 1468 |
+
else:
|
| 1469 |
+
tool_use = {
|
| 1470 |
+
"id": None,
|
| 1471 |
+
"type": "function",
|
| 1472 |
+
"function": {
|
| 1473 |
+
"name": None,
|
| 1474 |
+
"arguments": delta_obj["toolUse"]["input"],
|
| 1475 |
+
},
|
| 1476 |
+
"index": (
|
| 1477 |
+
self.tool_calls_index
|
| 1478 |
+
if self.tool_calls_index is not None
|
| 1479 |
+
else index
|
| 1480 |
+
),
|
| 1481 |
+
}
|
| 1482 |
+
elif "reasoningContent" in delta_obj:
|
| 1483 |
+
provider_specific_fields = {
|
| 1484 |
+
"reasoningContent": delta_obj["reasoningContent"],
|
| 1485 |
+
}
|
| 1486 |
+
reasoning_content = self.extract_reasoning_content_str(
|
| 1487 |
+
delta_obj["reasoningContent"]
|
| 1488 |
+
)
|
| 1489 |
+
thinking_blocks = self.translate_thinking_blocks(
|
| 1490 |
+
delta_obj["reasoningContent"]
|
| 1491 |
+
)
|
| 1492 |
+
if (
|
| 1493 |
+
thinking_blocks
|
| 1494 |
+
and len(thinking_blocks) > 0
|
| 1495 |
+
and reasoning_content is None
|
| 1496 |
+
):
|
| 1497 |
+
reasoning_content = (
|
| 1498 |
+
"" # set to non-empty string to ensure consistency with Anthropic
|
| 1499 |
+
)
|
| 1500 |
+
elif "citationsContent" in delta_obj:
|
| 1501 |
+
# Handle Nova grounding citations in streaming responses
|
| 1502 |
+
provider_specific_fields = {
|
| 1503 |
+
"citationsContent": delta_obj["citationsContent"],
|
| 1504 |
+
}
|
| 1505 |
+
return (
|
| 1506 |
+
text,
|
| 1507 |
+
tool_use,
|
| 1508 |
+
provider_specific_fields,
|
| 1509 |
+
reasoning_content,
|
| 1510 |
+
thinking_blocks,
|
| 1511 |
+
)
|
| 1512 |
+
|
| 1513 |
+
def _handle_converse_stop_event(
|
| 1514 |
+
self, index: int
|
| 1515 |
+
) -> Optional[ChatCompletionToolCallChunk]:
|
| 1516 |
+
"""Handle stop/contentBlockIndex event in converse chunk parsing."""
|
| 1517 |
+
tool_use: Optional[ChatCompletionToolCallChunk] = None
|
| 1518 |
+
|
| 1519 |
+
# If the ending block was the internal json_tool_call, skip emitting
|
| 1520 |
+
# the empty-args tool chunk and reset tracking state
|
| 1521 |
+
if (
|
| 1522 |
+
self.json_mode is True
|
| 1523 |
+
and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME
|
| 1524 |
+
):
|
| 1525 |
+
self._current_tool_name = None
|
| 1526 |
+
return tool_use
|
| 1527 |
+
|
| 1528 |
+
self._current_tool_name = None
|
| 1529 |
+
is_empty = self.check_empty_tool_call_args()
|
| 1530 |
+
if is_empty:
|
| 1531 |
+
tool_use = {
|
| 1532 |
+
"id": None,
|
| 1533 |
+
"type": "function",
|
| 1534 |
+
"function": {
|
| 1535 |
+
"name": None,
|
| 1536 |
+
"arguments": "{}",
|
| 1537 |
+
},
|
| 1538 |
+
"index": (
|
| 1539 |
+
self.tool_calls_index
|
| 1540 |
+
if self.tool_calls_index is not None
|
| 1541 |
+
else index
|
| 1542 |
+
),
|
| 1543 |
+
}
|
| 1544 |
+
return tool_use
|
| 1545 |
+
|
| 1546 |
+
def converse_chunk_parser(self, chunk_data: dict) -> ModelResponseStream:
|
| 1547 |
+
try:
|
| 1548 |
+
# Capture the conversationId from the first messageStart event
|
| 1549 |
+
# and use it as the consistent ID for all subsequent chunks.
|
| 1550 |
+
self._initialize_converse_response_id(chunk_data)
|
| 1551 |
+
|
| 1552 |
+
verbose_logger.debug("\n\nRaw Chunk: {}\n\n".format(chunk_data))
|
| 1553 |
+
text = ""
|
| 1554 |
+
tool_use: Optional[ChatCompletionToolCallChunk] = None
|
| 1555 |
+
finish_reason = ""
|
| 1556 |
+
usage: Optional[Usage] = None
|
| 1557 |
+
provider_specific_fields: dict = {}
|
| 1558 |
+
reasoning_content: Optional[str] = None
|
| 1559 |
+
thinking_blocks: Optional[
|
| 1560 |
+
List[
|
| 1561 |
+
Union[
|
| 1562 |
+
ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock
|
| 1563 |
+
]
|
| 1564 |
+
]
|
| 1565 |
+
] = None
|
| 1566 |
+
|
| 1567 |
+
content_block_index = int(chunk_data.get("contentBlockIndex", 0))
|
| 1568 |
+
if "start" in chunk_data:
|
| 1569 |
+
start_obj = ContentBlockStartEvent(**chunk_data["start"])
|
| 1570 |
+
(
|
| 1571 |
+
tool_use,
|
| 1572 |
+
provider_specific_fields,
|
| 1573 |
+
thinking_blocks,
|
| 1574 |
+
) = self._handle_converse_start_event(start_obj)
|
| 1575 |
+
elif "delta" in chunk_data:
|
| 1576 |
+
delta_obj = ContentBlockDeltaEvent(**chunk_data["delta"])
|
| 1577 |
+
(
|
| 1578 |
+
text,
|
| 1579 |
+
tool_use,
|
| 1580 |
+
provider_specific_fields,
|
| 1581 |
+
reasoning_content,
|
| 1582 |
+
thinking_blocks,
|
| 1583 |
+
) = self._handle_converse_delta_event(delta_obj, content_block_index)
|
| 1584 |
+
elif (
|
| 1585 |
+
"contentBlockIndex" in chunk_data
|
| 1586 |
+
): # stop block, no 'start' or 'delta' object
|
| 1587 |
+
tool_use = self._handle_converse_stop_event(content_block_index)
|
| 1588 |
+
elif "stopReason" in chunk_data:
|
| 1589 |
+
finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
|
| 1590 |
+
elif "usage" in chunk_data:
|
| 1591 |
+
usage = converse_config._transform_usage(chunk_data.get("usage", {}))
|
| 1592 |
+
|
| 1593 |
+
model_response_provider_specific_fields = {}
|
| 1594 |
+
if "trace" in chunk_data:
|
| 1595 |
+
trace = chunk_data.get("trace")
|
| 1596 |
+
model_response_provider_specific_fields["trace"] = trace
|
| 1597 |
+
response = ModelResponseStream(
|
| 1598 |
+
choices=[
|
| 1599 |
+
StreamingChoices(
|
| 1600 |
+
finish_reason=finish_reason,
|
| 1601 |
+
index=0, # Always 0 - Bedrock never returns multiple choices
|
| 1602 |
+
delta=Delta(
|
| 1603 |
+
content=text,
|
| 1604 |
+
role="assistant",
|
| 1605 |
+
tool_calls=[tool_use] if tool_use else None,
|
| 1606 |
+
provider_specific_fields=(
|
| 1607 |
+
provider_specific_fields
|
| 1608 |
+
if provider_specific_fields
|
| 1609 |
+
else None
|
| 1610 |
+
),
|
| 1611 |
+
thinking_blocks=thinking_blocks,
|
| 1612 |
+
reasoning_content=reasoning_content,
|
| 1613 |
+
),
|
| 1614 |
+
)
|
| 1615 |
+
],
|
| 1616 |
+
id=self.response_id,
|
| 1617 |
+
model=self.model,
|
| 1618 |
+
usage=usage,
|
| 1619 |
+
provider_specific_fields=model_response_provider_specific_fields,
|
| 1620 |
+
)
|
| 1621 |
+
|
| 1622 |
+
return response
|
| 1623 |
+
except Exception as e:
|
| 1624 |
+
raise Exception("Received streaming error - {}".format(str(e)))
|
| 1625 |
+
|
| 1626 |
+
def _chunk_parser(
|
| 1627 |
+
self, chunk_data: dict
|
| 1628 |
+
) -> Union[GChunk, ModelResponseStream, dict]:
|
| 1629 |
+
text = ""
|
| 1630 |
+
is_finished = False
|
| 1631 |
+
finish_reason = ""
|
| 1632 |
+
if "outputText" in chunk_data:
|
| 1633 |
+
text = chunk_data["outputText"]
|
| 1634 |
+
# ai21 mapping
|
| 1635 |
+
elif "ai21" in self.model: # fake ai21 streaming
|
| 1636 |
+
text = chunk_data.get("completions")[0].get("data").get("text") # type: ignore
|
| 1637 |
+
is_finished = True
|
| 1638 |
+
finish_reason = "stop"
|
| 1639 |
+
######## /bedrock/converse mappings ###############
|
| 1640 |
+
elif (
|
| 1641 |
+
"contentBlockIndex" in chunk_data
|
| 1642 |
+
or "stopReason" in chunk_data
|
| 1643 |
+
or "metrics" in chunk_data
|
| 1644 |
+
or "trace" in chunk_data
|
| 1645 |
+
):
|
| 1646 |
+
return self.converse_chunk_parser(chunk_data=chunk_data)
|
| 1647 |
+
######### /bedrock/invoke nova mappings ###############
|
| 1648 |
+
elif "contentBlockDelta" in chunk_data:
|
| 1649 |
+
# when using /bedrock/invoke/nova, the chunk_data is nested under "contentBlockDelta"
|
| 1650 |
+
_chunk_data = chunk_data.get("contentBlockDelta", {})
|
| 1651 |
+
return self.converse_chunk_parser(chunk_data=_chunk_data)
|
| 1652 |
+
######## bedrock.mistral mappings ###############
|
| 1653 |
+
elif "outputs" in chunk_data:
|
| 1654 |
+
if (
|
| 1655 |
+
len(chunk_data["outputs"]) == 1
|
| 1656 |
+
and chunk_data["outputs"][0].get("text", None) is not None
|
| 1657 |
+
):
|
| 1658 |
+
text = chunk_data["outputs"][0]["text"]
|
| 1659 |
+
stop_reason = chunk_data.get("stop_reason", None)
|
| 1660 |
+
if stop_reason is not None:
|
| 1661 |
+
is_finished = True
|
| 1662 |
+
finish_reason = stop_reason
|
| 1663 |
+
######## bedrock.cohere mappings ###############
|
| 1664 |
+
# meta mapping
|
| 1665 |
+
elif "generation" in chunk_data:
|
| 1666 |
+
text = chunk_data["generation"] # bedrock.meta
|
| 1667 |
+
# cohere mapping
|
| 1668 |
+
elif "text" in chunk_data:
|
| 1669 |
+
text = chunk_data["text"] # bedrock.cohere
|
| 1670 |
+
# cohere mapping for finish reason
|
| 1671 |
+
elif "finish_reason" in chunk_data:
|
| 1672 |
+
finish_reason = chunk_data["finish_reason"]
|
| 1673 |
+
is_finished = True
|
| 1674 |
+
elif chunk_data.get("completionReason", None):
|
| 1675 |
+
is_finished = True
|
| 1676 |
+
finish_reason = chunk_data["completionReason"]
|
| 1677 |
+
return GChunk(
|
| 1678 |
+
text=text,
|
| 1679 |
+
is_finished=is_finished,
|
| 1680 |
+
finish_reason=finish_reason,
|
| 1681 |
+
usage=None,
|
| 1682 |
+
index=0,
|
| 1683 |
+
tool_use=None,
|
| 1684 |
+
)
|
| 1685 |
+
|
| 1686 |
+
def iter_bytes(
|
| 1687 |
+
self, iterator: Iterator[bytes]
|
| 1688 |
+
) -> Iterator[Union[GChunk, ModelResponseStream, dict]]:
|
| 1689 |
+
"""Given an iterator that yields lines, iterate over it & yield every event encountered"""
|
| 1690 |
+
from botocore.eventstream import EventStreamBuffer
|
| 1691 |
+
|
| 1692 |
+
event_stream_buffer = EventStreamBuffer()
|
| 1693 |
+
for chunk in iterator:
|
| 1694 |
+
event_stream_buffer.add_data(chunk)
|
| 1695 |
+
for event in event_stream_buffer:
|
| 1696 |
+
message = self._parse_message_from_event(event)
|
| 1697 |
+
if message:
|
| 1698 |
+
# sse_event = ServerSentEvent(data=message, event="completion")
|
| 1699 |
+
_data = json.loads(message)
|
| 1700 |
+
yield self._chunk_parser(chunk_data=_data)
|
| 1701 |
+
|
| 1702 |
+
async def aiter_bytes(
|
| 1703 |
+
self, iterator: AsyncIterator[bytes]
|
| 1704 |
+
) -> AsyncIterator[Union[GChunk, ModelResponseStream, dict]]:
|
| 1705 |
+
"""Given an async iterator that yields lines, iterate over it & yield every event encountered"""
|
| 1706 |
+
from botocore.eventstream import EventStreamBuffer
|
| 1707 |
+
|
| 1708 |
+
event_stream_buffer = EventStreamBuffer()
|
| 1709 |
+
async for chunk in iterator:
|
| 1710 |
+
event_stream_buffer.add_data(chunk)
|
| 1711 |
+
for event in event_stream_buffer:
|
| 1712 |
+
message = self._parse_message_from_event(event)
|
| 1713 |
+
if message:
|
| 1714 |
+
_data = json.loads(message)
|
| 1715 |
+
yield self._chunk_parser(chunk_data=_data)
|
| 1716 |
+
|
| 1717 |
+
def _parse_message_from_event(self, event) -> Optional[str]:
|
| 1718 |
+
response_dict = event.to_response_dict()
|
| 1719 |
+
parsed_response = self.parser.parse(response_dict, get_response_stream_shape())
|
| 1720 |
+
|
| 1721 |
+
if response_dict["status_code"] != 200:
|
| 1722 |
+
decoded_body = response_dict["body"].decode()
|
| 1723 |
+
if isinstance(decoded_body, dict):
|
| 1724 |
+
error_message = decoded_body.get("message")
|
| 1725 |
+
elif isinstance(decoded_body, str):
|
| 1726 |
+
error_message = decoded_body
|
| 1727 |
+
else:
|
| 1728 |
+
error_message = ""
|
| 1729 |
+
exception_status = response_dict["headers"].get(":exception-type")
|
| 1730 |
+
error_message = exception_status + " " + error_message
|
| 1731 |
+
raise BedrockError(
|
| 1732 |
+
status_code=response_dict["status_code"],
|
| 1733 |
+
message=(
|
| 1734 |
+
json.dumps(error_message)
|
| 1735 |
+
if isinstance(error_message, dict)
|
| 1736 |
+
else error_message
|
| 1737 |
+
),
|
| 1738 |
+
)
|
| 1739 |
+
if "chunk" in parsed_response:
|
| 1740 |
+
chunk = parsed_response.get("chunk")
|
| 1741 |
+
if not chunk:
|
| 1742 |
+
return None
|
| 1743 |
+
return chunk.get("bytes").decode() # type: ignore[no-any-return]
|
| 1744 |
+
else:
|
| 1745 |
+
chunk = response_dict.get("body")
|
| 1746 |
+
if not chunk:
|
| 1747 |
+
return None
|
| 1748 |
+
|
| 1749 |
+
return chunk.decode() # type: ignore[no-any-return]
|
| 1750 |
+
|
| 1751 |
+
|
| 1752 |
+
class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder):
|
| 1753 |
+
def __init__(
|
| 1754 |
+
self,
|
| 1755 |
+
model: str,
|
| 1756 |
+
sync_stream: bool,
|
| 1757 |
+
json_mode: Optional[bool] = None,
|
| 1758 |
+
) -> None:
|
| 1759 |
+
"""
|
| 1760 |
+
Child class of AWSEventStreamDecoder that handles the streaming response from the Anthropic family of models
|
| 1761 |
+
|
| 1762 |
+
The only difference between AWSEventStreamDecoder and AmazonAnthropicClaudeStreamDecoder is the `chunk_parser` method
|
| 1763 |
+
"""
|
| 1764 |
+
super().__init__(model=model)
|
| 1765 |
+
self.anthropic_model_response_iterator = AnthropicModelResponseIterator(
|
| 1766 |
+
streaming_response=None,
|
| 1767 |
+
sync_stream=sync_stream,
|
| 1768 |
+
json_mode=json_mode,
|
| 1769 |
+
)
|
| 1770 |
+
|
| 1771 |
+
def _chunk_parser(self, chunk_data: dict) -> ModelResponseStream:
|
| 1772 |
+
return self.anthropic_model_response_iterator.chunk_parser(chunk=chunk_data)
|
| 1773 |
+
|
| 1774 |
+
|
| 1775 |
+
class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder):
|
| 1776 |
+
def __init__(
|
| 1777 |
+
self,
|
| 1778 |
+
model: str,
|
| 1779 |
+
sync_stream: bool,
|
| 1780 |
+
) -> None:
|
| 1781 |
+
super().__init__(model=model)
|
| 1782 |
+
from litellm.llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import (
|
| 1783 |
+
AmazonDeepseekR1ResponseIterator,
|
| 1784 |
+
)
|
| 1785 |
+
|
| 1786 |
+
self.deepseek_model_response_iterator = AmazonDeepseekR1ResponseIterator(
|
| 1787 |
+
streaming_response=None,
|
| 1788 |
+
sync_stream=sync_stream,
|
| 1789 |
+
)
|
| 1790 |
+
|
| 1791 |
+
def _chunk_parser(
|
| 1792 |
+
self, chunk_data: dict
|
| 1793 |
+
) -> Union[GChunk, ModelResponseStream, dict]:
|
| 1794 |
+
return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)
|
| 1795 |
+
|
| 1796 |
+
|
| 1797 |
+
class MockResponseIterator: # for returning ai21 streaming responses
|
| 1798 |
+
def __init__(self, model_response, json_mode: Optional[bool] = False):
|
| 1799 |
+
self.model_response = model_response
|
| 1800 |
+
self.json_mode = json_mode
|
| 1801 |
+
self.is_done = False
|
| 1802 |
+
|
| 1803 |
+
# Sync iterator
|
| 1804 |
+
def __iter__(self):
|
| 1805 |
+
return self
|
| 1806 |
+
|
| 1807 |
+
def _handle_json_mode_chunk(
|
| 1808 |
+
self, text: str, tool_calls: Optional[List[ChatCompletionToolCallChunk]]
|
| 1809 |
+
) -> Tuple[str, Optional[ChatCompletionToolCallChunk]]:
|
| 1810 |
+
"""
|
| 1811 |
+
If JSON mode is enabled, convert the tool call to a message.
|
| 1812 |
+
|
| 1813 |
+
Bedrock returns the JSON schema as part of the tool call
|
| 1814 |
+
OpenAI returns the JSON schema as part of the content, this handles placing it in the content
|
| 1815 |
+
|
| 1816 |
+
Args:
|
| 1817 |
+
text: str
|
| 1818 |
+
tool_use: Optional[ChatCompletionToolCallChunk]
|
| 1819 |
+
Returns:
|
| 1820 |
+
Tuple[str, Optional[ChatCompletionToolCallChunk]]
|
| 1821 |
+
|
| 1822 |
+
text: The text to use in the content
|
| 1823 |
+
tool_use: The ChatCompletionToolCallChunk to use in the chunk response
|
| 1824 |
+
"""
|
| 1825 |
+
tool_use: Optional[ChatCompletionToolCallChunk] = None
|
| 1826 |
+
if self.json_mode is True and tool_calls is not None:
|
| 1827 |
+
message = litellm.AnthropicConfig()._convert_tool_response_to_message(
|
| 1828 |
+
tool_calls=tool_calls
|
| 1829 |
+
)
|
| 1830 |
+
if message is not None:
|
| 1831 |
+
text = message.content or ""
|
| 1832 |
+
tool_use = None
|
| 1833 |
+
elif tool_calls is not None and len(tool_calls) > 0:
|
| 1834 |
+
tool_use = tool_calls[0]
|
| 1835 |
+
return text, tool_use
|
| 1836 |
+
|
| 1837 |
+
def _chunk_parser(self, chunk_data: ModelResponse) -> GChunk:
|
| 1838 |
+
try:
|
| 1839 |
+
chunk_usage: Usage = getattr(chunk_data, "usage")
|
| 1840 |
+
text = chunk_data.choices[0].message.content or "" # type: ignore
|
| 1841 |
+
tool_use = None
|
| 1842 |
+
_model_response_tool_call = cast(
|
| 1843 |
+
Optional[List[ChatCompletionMessageToolCall]],
|
| 1844 |
+
cast(Choices, chunk_data.choices[0]).message.tool_calls,
|
| 1845 |
+
)
|
| 1846 |
+
if self.json_mode is True:
|
| 1847 |
+
text, tool_use = self._handle_json_mode_chunk(
|
| 1848 |
+
text=text,
|
| 1849 |
+
tool_calls=chunk_data.choices[0].message.tool_calls, # type: ignore
|
| 1850 |
+
)
|
| 1851 |
+
elif _model_response_tool_call is not None:
|
| 1852 |
+
tool_use = ChatCompletionToolCallChunk(
|
| 1853 |
+
id=_model_response_tool_call[0].id,
|
| 1854 |
+
type="function",
|
| 1855 |
+
function=ChatCompletionToolCallFunctionChunk(
|
| 1856 |
+
name=_model_response_tool_call[0].function.name,
|
| 1857 |
+
arguments=_model_response_tool_call[0].function.arguments,
|
| 1858 |
+
),
|
| 1859 |
+
index=0,
|
| 1860 |
+
)
|
| 1861 |
+
processed_chunk = GChunk(
|
| 1862 |
+
text=text,
|
| 1863 |
+
tool_use=tool_use,
|
| 1864 |
+
is_finished=True,
|
| 1865 |
+
finish_reason=map_finish_reason(
|
| 1866 |
+
finish_reason=chunk_data.choices[0].finish_reason or ""
|
| 1867 |
+
),
|
| 1868 |
+
usage=ChatCompletionUsageBlock(
|
| 1869 |
+
prompt_tokens=chunk_usage.prompt_tokens,
|
| 1870 |
+
completion_tokens=chunk_usage.completion_tokens,
|
| 1871 |
+
total_tokens=chunk_usage.total_tokens,
|
| 1872 |
+
),
|
| 1873 |
+
index=0,
|
| 1874 |
+
)
|
| 1875 |
+
return processed_chunk
|
| 1876 |
+
except Exception as e:
|
| 1877 |
+
raise ValueError(f"Failed to decode chunk: {chunk_data}. Error: {e}")
|
| 1878 |
+
|
| 1879 |
+
def __next__(self):
|
| 1880 |
+
if self.is_done:
|
| 1881 |
+
raise StopIteration
|
| 1882 |
+
self.is_done = True
|
| 1883 |
+
return self._chunk_parser(self.model_response)
|
| 1884 |
+
|
| 1885 |
+
# Async iterator
|
| 1886 |
+
def __aiter__(self):
|
| 1887 |
+
return self
|
| 1888 |
+
|
| 1889 |
+
async def __anext__(self):
|
| 1890 |
+
if self.is_done:
|
| 1891 |
+
raise StopAsyncIteration
|
| 1892 |
+
self.is_done = True
|
| 1893 |
+
return self._chunk_parser(self.model_response)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import types
|
| 2 |
+
from typing import List, Optional
|
| 3 |
+
|
| 4 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
| 5 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 6 |
+
AmazonInvokeConfig,
|
| 7 |
+
)
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class AmazonAI21Config(AmazonInvokeConfig, BaseConfig):
|
| 11 |
+
"""
|
| 12 |
+
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=j2-ultra
|
| 13 |
+
|
| 14 |
+
Supported Params for the Amazon / AI21 models:
|
| 15 |
+
|
| 16 |
+
- `maxTokens` (int32): The maximum number of tokens to generate per result. Optional, default is 16. If no `stopSequences` are given, generation stops after producing `maxTokens`.
|
| 17 |
+
|
| 18 |
+
- `temperature` (float): Modifies the distribution from which tokens are sampled. Optional, default is 0.7. A value of 0 essentially disables sampling and results in greedy decoding.
|
| 19 |
+
|
| 20 |
+
- `topP` (float): Used for sampling tokens from the corresponding top percentile of probability mass. Optional, default is 1. For instance, a value of 0.9 considers only tokens comprising the top 90% probability mass.
|
| 21 |
+
|
| 22 |
+
- `stopSequences` (array of strings): Stops decoding if any of the input strings is generated. Optional.
|
| 23 |
+
|
| 24 |
+
- `frequencyPenalty` (object): Placeholder for frequency penalty object.
|
| 25 |
+
|
| 26 |
+
- `presencePenalty` (object): Placeholder for presence penalty object.
|
| 27 |
+
|
| 28 |
+
- `countPenalty` (object): Placeholder for count penalty object.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
maxTokens: Optional[int] = None
|
| 32 |
+
temperature: Optional[float] = None
|
| 33 |
+
topP: Optional[float] = None
|
| 34 |
+
stopSequences: Optional[list] = None
|
| 35 |
+
frequencePenalty: Optional[dict] = None
|
| 36 |
+
presencePenalty: Optional[dict] = None
|
| 37 |
+
countPenalty: Optional[dict] = None
|
| 38 |
+
|
| 39 |
+
def __init__(
|
| 40 |
+
self,
|
| 41 |
+
maxTokens: Optional[int] = None,
|
| 42 |
+
temperature: Optional[float] = None,
|
| 43 |
+
topP: Optional[float] = None,
|
| 44 |
+
stopSequences: Optional[list] = None,
|
| 45 |
+
frequencePenalty: Optional[dict] = None,
|
| 46 |
+
presencePenalty: Optional[dict] = None,
|
| 47 |
+
countPenalty: Optional[dict] = None,
|
| 48 |
+
) -> None:
|
| 49 |
+
locals_ = locals().copy()
|
| 50 |
+
for key, value in locals_.items():
|
| 51 |
+
if key != "self" and value is not None:
|
| 52 |
+
setattr(self.__class__, key, value)
|
| 53 |
+
|
| 54 |
+
AmazonInvokeConfig.__init__(self)
|
| 55 |
+
|
| 56 |
+
@classmethod
|
| 57 |
+
def get_config(cls):
|
| 58 |
+
return {
|
| 59 |
+
k: v
|
| 60 |
+
for k, v in cls.__dict__.items()
|
| 61 |
+
if not k.startswith("__")
|
| 62 |
+
and not k.startswith("_abc")
|
| 63 |
+
and not isinstance(
|
| 64 |
+
v,
|
| 65 |
+
(
|
| 66 |
+
types.FunctionType,
|
| 67 |
+
types.BuiltinFunctionType,
|
| 68 |
+
classmethod,
|
| 69 |
+
staticmethod,
|
| 70 |
+
),
|
| 71 |
+
)
|
| 72 |
+
and v is not None
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
def get_supported_openai_params(self, model: str) -> List:
|
| 76 |
+
return [
|
| 77 |
+
"max_tokens",
|
| 78 |
+
"temperature",
|
| 79 |
+
"top_p",
|
| 80 |
+
"stream",
|
| 81 |
+
]
|
| 82 |
+
|
| 83 |
+
def map_openai_params(
|
| 84 |
+
self,
|
| 85 |
+
non_default_params: dict,
|
| 86 |
+
optional_params: dict,
|
| 87 |
+
model: str,
|
| 88 |
+
drop_params: bool,
|
| 89 |
+
) -> dict:
|
| 90 |
+
for k, v in non_default_params.items():
|
| 91 |
+
if k == "max_tokens":
|
| 92 |
+
optional_params["maxTokens"] = v
|
| 93 |
+
if k == "temperature":
|
| 94 |
+
optional_params["temperature"] = v
|
| 95 |
+
if k == "top_p":
|
| 96 |
+
optional_params["topP"] = v
|
| 97 |
+
if k == "stream":
|
| 98 |
+
optional_params["stream"] = v
|
| 99 |
+
return optional_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import types
|
| 2 |
+
from typing import List, Optional
|
| 3 |
+
|
| 4 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 5 |
+
AmazonInvokeConfig,
|
| 6 |
+
)
|
| 7 |
+
from litellm.llms.cohere.chat.transformation import CohereChatConfig
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class AmazonCohereConfig(AmazonInvokeConfig, CohereChatConfig):
|
| 11 |
+
"""
|
| 12 |
+
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=command
|
| 13 |
+
|
| 14 |
+
Supported Params for the Amazon / Cohere models:
|
| 15 |
+
|
| 16 |
+
- `max_tokens` (integer) max tokens,
|
| 17 |
+
- `temperature` (float) model temperature,
|
| 18 |
+
- `return_likelihood` (string) n/a
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
max_tokens: Optional[int] = None
|
| 22 |
+
return_likelihood: Optional[str] = None
|
| 23 |
+
|
| 24 |
+
def __init__(
|
| 25 |
+
self,
|
| 26 |
+
max_tokens: Optional[int] = None,
|
| 27 |
+
temperature: Optional[float] = None,
|
| 28 |
+
return_likelihood: Optional[str] = None,
|
| 29 |
+
) -> None:
|
| 30 |
+
locals_ = locals().copy()
|
| 31 |
+
for key, value in locals_.items():
|
| 32 |
+
if key != "self" and value is not None:
|
| 33 |
+
setattr(self.__class__, key, value)
|
| 34 |
+
|
| 35 |
+
AmazonInvokeConfig.__init__(self)
|
| 36 |
+
|
| 37 |
+
@classmethod
|
| 38 |
+
def get_config(cls):
|
| 39 |
+
return {
|
| 40 |
+
k: v
|
| 41 |
+
for k, v in cls.__dict__.items()
|
| 42 |
+
if not k.startswith("__")
|
| 43 |
+
and not k.startswith("_abc")
|
| 44 |
+
and not isinstance(
|
| 45 |
+
v,
|
| 46 |
+
(
|
| 47 |
+
types.FunctionType,
|
| 48 |
+
types.BuiltinFunctionType,
|
| 49 |
+
classmethod,
|
| 50 |
+
staticmethod,
|
| 51 |
+
),
|
| 52 |
+
)
|
| 53 |
+
and v is not None
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 57 |
+
supported_params = CohereChatConfig.get_supported_openai_params(
|
| 58 |
+
self, model=model
|
| 59 |
+
)
|
| 60 |
+
return supported_params
|
| 61 |
+
|
| 62 |
+
def map_openai_params(
|
| 63 |
+
self,
|
| 64 |
+
non_default_params: dict,
|
| 65 |
+
optional_params: dict,
|
| 66 |
+
model: str,
|
| 67 |
+
drop_params: bool,
|
| 68 |
+
) -> dict:
|
| 69 |
+
return CohereChatConfig.map_openai_params(
|
| 70 |
+
self,
|
| 71 |
+
non_default_params=non_default_params,
|
| 72 |
+
optional_params=optional_params,
|
| 73 |
+
model=model,
|
| 74 |
+
drop_params=drop_params,
|
| 75 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, List, Optional, cast
|
| 2 |
+
|
| 3 |
+
from httpx import Response
|
| 4 |
+
|
| 5 |
+
from litellm import verbose_logger
|
| 6 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 7 |
+
_parse_content_for_reasoning,
|
| 8 |
+
)
|
| 9 |
+
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
| 10 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 11 |
+
LiteLLMLoggingObj,
|
| 12 |
+
)
|
| 13 |
+
from litellm.types.llms.bedrock import AmazonDeepSeekR1StreamingResponse
|
| 14 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 15 |
+
from litellm.types.utils import (
|
| 16 |
+
ChatCompletionUsageBlock,
|
| 17 |
+
Choices,
|
| 18 |
+
Delta,
|
| 19 |
+
Message,
|
| 20 |
+
ModelResponse,
|
| 21 |
+
ModelResponseStream,
|
| 22 |
+
StreamingChoices,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
from .amazon_llama_transformation import AmazonLlamaConfig
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class AmazonDeepSeekR1Config(AmazonLlamaConfig):
|
| 29 |
+
def transform_response(
|
| 30 |
+
self,
|
| 31 |
+
model: str,
|
| 32 |
+
raw_response: Response,
|
| 33 |
+
model_response: ModelResponse,
|
| 34 |
+
logging_obj: LiteLLMLoggingObj,
|
| 35 |
+
request_data: dict,
|
| 36 |
+
messages: List[AllMessageValues],
|
| 37 |
+
optional_params: dict,
|
| 38 |
+
litellm_params: dict,
|
| 39 |
+
encoding: Any,
|
| 40 |
+
api_key: Optional[str] = None,
|
| 41 |
+
json_mode: Optional[bool] = None,
|
| 42 |
+
) -> ModelResponse:
|
| 43 |
+
"""
|
| 44 |
+
Extract the reasoning content, and return it as a separate field in the response.
|
| 45 |
+
"""
|
| 46 |
+
response = super().transform_response(
|
| 47 |
+
model,
|
| 48 |
+
raw_response,
|
| 49 |
+
model_response,
|
| 50 |
+
logging_obj,
|
| 51 |
+
request_data,
|
| 52 |
+
messages,
|
| 53 |
+
optional_params,
|
| 54 |
+
litellm_params,
|
| 55 |
+
encoding,
|
| 56 |
+
api_key,
|
| 57 |
+
json_mode,
|
| 58 |
+
)
|
| 59 |
+
prompt = cast(Optional[str], request_data.get("prompt"))
|
| 60 |
+
message_content = cast(
|
| 61 |
+
Optional[str], cast(Choices, response.choices[0]).message.get("content")
|
| 62 |
+
)
|
| 63 |
+
if prompt and prompt.strip().endswith("<think>") and message_content:
|
| 64 |
+
message_content_with_reasoning_token = "<think>" + message_content
|
| 65 |
+
reasoning, content = _parse_content_for_reasoning(
|
| 66 |
+
message_content_with_reasoning_token
|
| 67 |
+
)
|
| 68 |
+
provider_specific_fields = (
|
| 69 |
+
cast(Choices, response.choices[0]).message.provider_specific_fields
|
| 70 |
+
or {}
|
| 71 |
+
)
|
| 72 |
+
if reasoning:
|
| 73 |
+
provider_specific_fields["reasoning_content"] = reasoning
|
| 74 |
+
|
| 75 |
+
message = Message(
|
| 76 |
+
**{
|
| 77 |
+
**cast(Choices, response.choices[0]).message.model_dump(),
|
| 78 |
+
"content": content,
|
| 79 |
+
"provider_specific_fields": provider_specific_fields,
|
| 80 |
+
}
|
| 81 |
+
)
|
| 82 |
+
cast(Choices, response.choices[0]).message = message
|
| 83 |
+
return response
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class AmazonDeepseekR1ResponseIterator(BaseModelResponseIterator):
|
| 87 |
+
def __init__(self, streaming_response: Any, sync_stream: bool) -> None:
|
| 88 |
+
super().__init__(streaming_response=streaming_response, sync_stream=sync_stream)
|
| 89 |
+
self.has_finished_thinking = False
|
| 90 |
+
|
| 91 |
+
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
|
| 92 |
+
"""
|
| 93 |
+
Deepseek r1 starts by thinking, then it generates the response.
|
| 94 |
+
"""
|
| 95 |
+
try:
|
| 96 |
+
typed_chunk = AmazonDeepSeekR1StreamingResponse(**chunk) # type: ignore
|
| 97 |
+
generated_content = typed_chunk["generation"]
|
| 98 |
+
if generated_content == "</think>" and not self.has_finished_thinking:
|
| 99 |
+
verbose_logger.debug(
|
| 100 |
+
"Deepseek r1: </think> received, setting has_finished_thinking to True"
|
| 101 |
+
)
|
| 102 |
+
generated_content = ""
|
| 103 |
+
self.has_finished_thinking = True
|
| 104 |
+
|
| 105 |
+
prompt_token_count = typed_chunk.get("prompt_token_count") or 0
|
| 106 |
+
generation_token_count = typed_chunk.get("generation_token_count") or 0
|
| 107 |
+
usage = ChatCompletionUsageBlock(
|
| 108 |
+
prompt_tokens=prompt_token_count,
|
| 109 |
+
completion_tokens=generation_token_count,
|
| 110 |
+
total_tokens=prompt_token_count + generation_token_count,
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
return ModelResponseStream(
|
| 114 |
+
choices=[
|
| 115 |
+
StreamingChoices(
|
| 116 |
+
finish_reason=typed_chunk["stop_reason"],
|
| 117 |
+
delta=Delta(
|
| 118 |
+
content=(
|
| 119 |
+
generated_content
|
| 120 |
+
if self.has_finished_thinking
|
| 121 |
+
else None
|
| 122 |
+
),
|
| 123 |
+
reasoning_content=(
|
| 124 |
+
generated_content
|
| 125 |
+
if not self.has_finished_thinking
|
| 126 |
+
else None
|
| 127 |
+
),
|
| 128 |
+
),
|
| 129 |
+
)
|
| 130 |
+
],
|
| 131 |
+
usage=usage,
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
except Exception as e:
|
| 135 |
+
raise e
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import types
|
| 2 |
+
from typing import List, Optional
|
| 3 |
+
|
| 4 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
| 5 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 6 |
+
AmazonInvokeConfig,
|
| 7 |
+
)
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class AmazonLlamaConfig(AmazonInvokeConfig, BaseConfig):
|
| 11 |
+
"""
|
| 12 |
+
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=meta.llama2-13b-chat-v1
|
| 13 |
+
|
| 14 |
+
Supported Params for the Amazon / Meta Llama models:
|
| 15 |
+
|
| 16 |
+
- `max_gen_len` (integer) max tokens,
|
| 17 |
+
- `temperature` (float) temperature for model,
|
| 18 |
+
- `top_p` (float) top p for model
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
max_gen_len: Optional[int] = None
|
| 22 |
+
temperature: Optional[float] = None
|
| 23 |
+
topP: Optional[float] = None
|
| 24 |
+
|
| 25 |
+
def __init__(
|
| 26 |
+
self,
|
| 27 |
+
maxTokenCount: Optional[int] = None,
|
| 28 |
+
temperature: Optional[float] = None,
|
| 29 |
+
topP: Optional[int] = None,
|
| 30 |
+
) -> None:
|
| 31 |
+
locals_ = locals().copy()
|
| 32 |
+
for key, value in locals_.items():
|
| 33 |
+
if key != "self" and value is not None:
|
| 34 |
+
setattr(self.__class__, key, value)
|
| 35 |
+
AmazonInvokeConfig.__init__(self)
|
| 36 |
+
|
| 37 |
+
@classmethod
|
| 38 |
+
def get_config(cls):
|
| 39 |
+
return {
|
| 40 |
+
k: v
|
| 41 |
+
for k, v in cls.__dict__.items()
|
| 42 |
+
if not k.startswith("__")
|
| 43 |
+
and not k.startswith("_abc")
|
| 44 |
+
and not isinstance(
|
| 45 |
+
v,
|
| 46 |
+
(
|
| 47 |
+
types.FunctionType,
|
| 48 |
+
types.BuiltinFunctionType,
|
| 49 |
+
classmethod,
|
| 50 |
+
staticmethod,
|
| 51 |
+
),
|
| 52 |
+
)
|
| 53 |
+
and v is not None
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
def get_supported_openai_params(self, model: str) -> List:
|
| 57 |
+
return [
|
| 58 |
+
"max_tokens",
|
| 59 |
+
"temperature",
|
| 60 |
+
"top_p",
|
| 61 |
+
"stream",
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
def map_openai_params(
|
| 65 |
+
self,
|
| 66 |
+
non_default_params: dict,
|
| 67 |
+
optional_params: dict,
|
| 68 |
+
model: str,
|
| 69 |
+
drop_params: bool,
|
| 70 |
+
) -> dict:
|
| 71 |
+
for k, v in non_default_params.items():
|
| 72 |
+
if k == "max_tokens":
|
| 73 |
+
optional_params["max_gen_len"] = v
|
| 74 |
+
if k == "temperature":
|
| 75 |
+
optional_params["temperature"] = v
|
| 76 |
+
if k == "top_p":
|
| 77 |
+
optional_params["top_p"] = v
|
| 78 |
+
if k == "stream":
|
| 79 |
+
optional_params["stream"] = v
|
| 80 |
+
return optional_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import types
|
| 2 |
+
from typing import List, Optional, TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
| 5 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 6 |
+
AmazonInvokeConfig,
|
| 7 |
+
)
|
| 8 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 9 |
+
|
| 10 |
+
if TYPE_CHECKING:
|
| 11 |
+
from litellm.types.utils import ModelResponse
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class AmazonMistralConfig(AmazonInvokeConfig, BaseConfig):
|
| 15 |
+
"""
|
| 16 |
+
Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mistral.html
|
| 17 |
+
Supported Params for the Amazon / Mistral models:
|
| 18 |
+
|
| 19 |
+
- `max_tokens` (integer) max tokens,
|
| 20 |
+
- `temperature` (float) temperature for model,
|
| 21 |
+
- `top_p` (float) top p for model
|
| 22 |
+
- `stop` [string] A list of stop sequences that if generated by the model, stops the model from generating further output.
|
| 23 |
+
- `top_k` (float) top k for model
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
max_tokens: Optional[int] = None
|
| 27 |
+
temperature: Optional[float] = None
|
| 28 |
+
top_p: Optional[float] = None
|
| 29 |
+
top_k: Optional[float] = None
|
| 30 |
+
stop: Optional[List[str]] = None
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
max_tokens: Optional[int] = None,
|
| 35 |
+
temperature: Optional[float] = None,
|
| 36 |
+
top_p: Optional[int] = None,
|
| 37 |
+
top_k: Optional[float] = None,
|
| 38 |
+
stop: Optional[List[str]] = None,
|
| 39 |
+
) -> None:
|
| 40 |
+
locals_ = locals().copy()
|
| 41 |
+
for key, value in locals_.items():
|
| 42 |
+
if key != "self" and value is not None:
|
| 43 |
+
setattr(self.__class__, key, value)
|
| 44 |
+
|
| 45 |
+
AmazonInvokeConfig.__init__(self)
|
| 46 |
+
|
| 47 |
+
@classmethod
|
| 48 |
+
def get_config(cls):
|
| 49 |
+
return {
|
| 50 |
+
k: v
|
| 51 |
+
for k, v in cls.__dict__.items()
|
| 52 |
+
if not k.startswith("__")
|
| 53 |
+
and not k.startswith("_abc")
|
| 54 |
+
and not isinstance(
|
| 55 |
+
v,
|
| 56 |
+
(
|
| 57 |
+
types.FunctionType,
|
| 58 |
+
types.BuiltinFunctionType,
|
| 59 |
+
classmethod,
|
| 60 |
+
staticmethod,
|
| 61 |
+
),
|
| 62 |
+
)
|
| 63 |
+
and v is not None
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 67 |
+
return ["max_tokens", "temperature", "top_p", "stop", "stream"]
|
| 68 |
+
|
| 69 |
+
def map_openai_params(
|
| 70 |
+
self,
|
| 71 |
+
non_default_params: dict,
|
| 72 |
+
optional_params: dict,
|
| 73 |
+
model: str,
|
| 74 |
+
drop_params: bool,
|
| 75 |
+
) -> dict:
|
| 76 |
+
for k, v in non_default_params.items():
|
| 77 |
+
if k == "max_tokens":
|
| 78 |
+
optional_params["max_tokens"] = v
|
| 79 |
+
if k == "temperature":
|
| 80 |
+
optional_params["temperature"] = v
|
| 81 |
+
if k == "top_p":
|
| 82 |
+
optional_params["top_p"] = v
|
| 83 |
+
if k == "stop":
|
| 84 |
+
optional_params["stop"] = v
|
| 85 |
+
if k == "stream":
|
| 86 |
+
optional_params["stream"] = v
|
| 87 |
+
return optional_params
|
| 88 |
+
|
| 89 |
+
@staticmethod
|
| 90 |
+
def get_outputText(
|
| 91 |
+
completion_response: dict, model_response: "ModelResponse"
|
| 92 |
+
) -> str:
|
| 93 |
+
"""This function extracts the output text from a bedrock mistral completion.
|
| 94 |
+
As a side effect, it updates the finish reason for a model response.
|
| 95 |
+
|
| 96 |
+
Args:
|
| 97 |
+
completion_response: JSON from the completion.
|
| 98 |
+
model_response: ModelResponse
|
| 99 |
+
|
| 100 |
+
Returns:
|
| 101 |
+
A string with the response of the LLM
|
| 102 |
+
|
| 103 |
+
"""
|
| 104 |
+
if "choices" in completion_response:
|
| 105 |
+
outputText = completion_response["choices"][0]["message"]["content"]
|
| 106 |
+
model_response.choices[0].finish_reason = completion_response["choices"][0][
|
| 107 |
+
"finish_reason"
|
| 108 |
+
]
|
| 109 |
+
elif "outputs" in completion_response:
|
| 110 |
+
outputText = completion_response["outputs"][0]["text"]
|
| 111 |
+
model_response.choices[0].finish_reason = completion_response["outputs"][0][
|
| 112 |
+
"stop_reason"
|
| 113 |
+
]
|
| 114 |
+
else:
|
| 115 |
+
raise BedrockError(
|
| 116 |
+
message="Unexpected mistral completion response", status_code=400
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
return outputText
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation for Bedrock Moonshot AI (Kimi K2) models.
|
| 3 |
+
|
| 4 |
+
Supports the Kimi K2 Thinking model available on Amazon Bedrock.
|
| 5 |
+
Model format: bedrock/moonshot.kimi-k2-thinking-v1:0
|
| 6 |
+
|
| 7 |
+
Reference: https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from typing import TYPE_CHECKING, Any, List, Optional, Union
|
| 11 |
+
import re
|
| 12 |
+
|
| 13 |
+
import httpx
|
| 14 |
+
|
| 15 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 16 |
+
AmazonInvokeConfig,
|
| 17 |
+
)
|
| 18 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 19 |
+
from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig
|
| 20 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 21 |
+
from litellm.types.utils import Choices
|
| 22 |
+
|
| 23 |
+
if TYPE_CHECKING:
|
| 24 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 25 |
+
from litellm.types.utils import ModelResponse
|
| 26 |
+
|
| 27 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 28 |
+
else:
|
| 29 |
+
LiteLLMLoggingObj = Any
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
|
| 33 |
+
"""
|
| 34 |
+
Configuration for Bedrock Moonshot AI (Kimi K2) models.
|
| 35 |
+
|
| 36 |
+
Reference:
|
| 37 |
+
https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/
|
| 38 |
+
https://platform.moonshot.ai/docs/api/chat
|
| 39 |
+
|
| 40 |
+
Supported Params for the Amazon / Moonshot models:
|
| 41 |
+
- `max_tokens` (integer) max tokens
|
| 42 |
+
- `temperature` (float) temperature for model (0-1 for Moonshot)
|
| 43 |
+
- `top_p` (float) top p for model
|
| 44 |
+
- `stream` (bool) whether to stream responses
|
| 45 |
+
- `tools` (list) tool definitions (supported on kimi-k2-thinking)
|
| 46 |
+
- `tool_choice` (str|dict) tool choice specification (supported on kimi-k2-thinking)
|
| 47 |
+
|
| 48 |
+
NOT Supported on Bedrock:
|
| 49 |
+
- `stop` sequences (Bedrock doesn't support stopSequences field for this model)
|
| 50 |
+
|
| 51 |
+
Note: The kimi-k2-thinking model DOES support tool calls, unlike kimi-thinking-preview.
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
def __init__(self, **kwargs):
|
| 55 |
+
AmazonInvokeConfig.__init__(self, **kwargs)
|
| 56 |
+
MoonshotChatConfig.__init__(self, **kwargs)
|
| 57 |
+
|
| 58 |
+
@property
|
| 59 |
+
def custom_llm_provider(self) -> Optional[str]:
|
| 60 |
+
return "bedrock"
|
| 61 |
+
|
| 62 |
+
def _get_model_id(self, model: str) -> str:
|
| 63 |
+
"""
|
| 64 |
+
Extract the actual model ID from the LiteLLM model name.
|
| 65 |
+
|
| 66 |
+
Removes routing prefixes like:
|
| 67 |
+
- bedrock/invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
|
| 68 |
+
- invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
|
| 69 |
+
- moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
|
| 70 |
+
"""
|
| 71 |
+
# Remove bedrock/ prefix if present
|
| 72 |
+
if model.startswith("bedrock/"):
|
| 73 |
+
model = model[8:]
|
| 74 |
+
|
| 75 |
+
# Remove invoke/ prefix if present
|
| 76 |
+
if model.startswith("invoke/"):
|
| 77 |
+
model = model[7:]
|
| 78 |
+
|
| 79 |
+
# Remove any provider prefix (e.g., moonshot/)
|
| 80 |
+
if "/" in model and not model.startswith("arn:"):
|
| 81 |
+
parts = model.split("/", 1)
|
| 82 |
+
if len(parts) == 2:
|
| 83 |
+
model = parts[1]
|
| 84 |
+
|
| 85 |
+
return model
|
| 86 |
+
|
| 87 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 88 |
+
"""
|
| 89 |
+
Get the supported OpenAI params for Moonshot AI models on Bedrock.
|
| 90 |
+
|
| 91 |
+
Bedrock-specific limitations:
|
| 92 |
+
- stopSequences field is not supported on Bedrock (unlike native Moonshot API)
|
| 93 |
+
- functions parameter is not supported (use tools instead)
|
| 94 |
+
- tool_choice doesn't support "required" value
|
| 95 |
+
|
| 96 |
+
Note: kimi-k2-thinking DOES support tool calls (unlike kimi-thinking-preview)
|
| 97 |
+
The parent MoonshotChatConfig class handles the kimi-thinking-preview exclusion.
|
| 98 |
+
"""
|
| 99 |
+
excluded_params: List[str] = [
|
| 100 |
+
"functions",
|
| 101 |
+
"stop",
|
| 102 |
+
] # Bedrock doesn't support stopSequences
|
| 103 |
+
|
| 104 |
+
base_openai_params = super(
|
| 105 |
+
MoonshotChatConfig, self
|
| 106 |
+
).get_supported_openai_params(model=model)
|
| 107 |
+
final_params: List[str] = []
|
| 108 |
+
for param in base_openai_params:
|
| 109 |
+
if param not in excluded_params:
|
| 110 |
+
final_params.append(param)
|
| 111 |
+
|
| 112 |
+
return final_params
|
| 113 |
+
|
| 114 |
+
def map_openai_params(
|
| 115 |
+
self,
|
| 116 |
+
non_default_params: dict,
|
| 117 |
+
optional_params: dict,
|
| 118 |
+
model: str,
|
| 119 |
+
drop_params: bool,
|
| 120 |
+
) -> dict:
|
| 121 |
+
"""
|
| 122 |
+
Map OpenAI parameters to Moonshot AI parameters for Bedrock.
|
| 123 |
+
|
| 124 |
+
Handles Moonshot AI specific limitations:
|
| 125 |
+
- tool_choice doesn't support "required" value
|
| 126 |
+
- Temperature <0.3 limitation for n>1
|
| 127 |
+
- Temperature range is [0, 1] (not [0, 2] like OpenAI)
|
| 128 |
+
"""
|
| 129 |
+
return MoonshotChatConfig.map_openai_params(
|
| 130 |
+
self,
|
| 131 |
+
non_default_params=non_default_params,
|
| 132 |
+
optional_params=optional_params,
|
| 133 |
+
model=model,
|
| 134 |
+
drop_params=drop_params,
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
def transform_request(
|
| 138 |
+
self,
|
| 139 |
+
model: str,
|
| 140 |
+
messages: List[AllMessageValues],
|
| 141 |
+
optional_params: dict,
|
| 142 |
+
litellm_params: dict,
|
| 143 |
+
headers: dict,
|
| 144 |
+
) -> dict:
|
| 145 |
+
"""
|
| 146 |
+
Transform the request for Bedrock Moonshot AI models.
|
| 147 |
+
|
| 148 |
+
Uses the Moonshot transformation logic which handles:
|
| 149 |
+
- Converting content lists to strings (Moonshot doesn't support list format)
|
| 150 |
+
- Adding tool_choice="required" message if needed
|
| 151 |
+
- Temperature and parameter validation
|
| 152 |
+
|
| 153 |
+
"""
|
| 154 |
+
# Filter out AWS credentials using the existing method from BaseAWSLLM
|
| 155 |
+
self._get_boto_credentials_from_optional_params(optional_params, model)
|
| 156 |
+
|
| 157 |
+
# Strip routing prefixes to get the actual model ID
|
| 158 |
+
clean_model_id = self._get_model_id(model)
|
| 159 |
+
|
| 160 |
+
# Use Moonshot's transform_request which handles message transformation
|
| 161 |
+
# and tool_choice="required" workaround
|
| 162 |
+
return MoonshotChatConfig.transform_request(
|
| 163 |
+
self,
|
| 164 |
+
model=clean_model_id,
|
| 165 |
+
messages=messages,
|
| 166 |
+
optional_params=optional_params,
|
| 167 |
+
litellm_params=litellm_params,
|
| 168 |
+
headers=headers,
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
def _extract_reasoning_from_content(
|
| 172 |
+
self, content: str
|
| 173 |
+
) -> tuple[Optional[str], str]:
|
| 174 |
+
"""
|
| 175 |
+
Extract reasoning content from <reasoning> tags in the response.
|
| 176 |
+
|
| 177 |
+
Moonshot AI's Kimi K2 Thinking model returns reasoning in <reasoning> tags.
|
| 178 |
+
This method extracts that content and returns it separately.
|
| 179 |
+
|
| 180 |
+
Args:
|
| 181 |
+
content: The full content string from the API response
|
| 182 |
+
|
| 183 |
+
Returns:
|
| 184 |
+
tuple: (reasoning_content, main_content)
|
| 185 |
+
"""
|
| 186 |
+
if not content:
|
| 187 |
+
return None, content
|
| 188 |
+
|
| 189 |
+
# Match <reasoning>...</reasoning> tags
|
| 190 |
+
reasoning_match = re.match(
|
| 191 |
+
r"<reasoning>(.*?)</reasoning>\s*(.*)", content, re.DOTALL
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
if reasoning_match:
|
| 195 |
+
reasoning_content = reasoning_match.group(1).strip()
|
| 196 |
+
main_content = reasoning_match.group(2).strip()
|
| 197 |
+
return reasoning_content, main_content
|
| 198 |
+
|
| 199 |
+
return None, content
|
| 200 |
+
|
| 201 |
+
def transform_response(
|
| 202 |
+
self,
|
| 203 |
+
model: str,
|
| 204 |
+
raw_response: httpx.Response,
|
| 205 |
+
model_response: "ModelResponse",
|
| 206 |
+
logging_obj: LiteLLMLoggingObj,
|
| 207 |
+
request_data: dict,
|
| 208 |
+
messages: List[AllMessageValues],
|
| 209 |
+
optional_params: dict,
|
| 210 |
+
litellm_params: dict,
|
| 211 |
+
encoding: Any,
|
| 212 |
+
api_key: Optional[str] = None,
|
| 213 |
+
json_mode: Optional[bool] = None,
|
| 214 |
+
) -> "ModelResponse":
|
| 215 |
+
"""
|
| 216 |
+
Transform the response from Bedrock Moonshot AI models.
|
| 217 |
+
|
| 218 |
+
Moonshot AI uses OpenAI-compatible response format, but returns reasoning
|
| 219 |
+
content in <reasoning> tags. This method:
|
| 220 |
+
1. Calls parent class transformation
|
| 221 |
+
2. Extracts reasoning content from <reasoning> tags
|
| 222 |
+
3. Sets reasoning_content on the message object
|
| 223 |
+
"""
|
| 224 |
+
# First, get the standard transformation
|
| 225 |
+
model_response = MoonshotChatConfig.transform_response(
|
| 226 |
+
self,
|
| 227 |
+
model=model,
|
| 228 |
+
raw_response=raw_response,
|
| 229 |
+
model_response=model_response,
|
| 230 |
+
logging_obj=logging_obj,
|
| 231 |
+
request_data=request_data,
|
| 232 |
+
messages=messages,
|
| 233 |
+
optional_params=optional_params,
|
| 234 |
+
litellm_params=litellm_params,
|
| 235 |
+
encoding=encoding,
|
| 236 |
+
api_key=api_key,
|
| 237 |
+
json_mode=json_mode,
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
# Extract reasoning content from <reasoning> tags
|
| 241 |
+
if model_response.choices and len(model_response.choices) > 0:
|
| 242 |
+
for choice in model_response.choices:
|
| 243 |
+
# Only process Choices (not StreamingChoices) which have message attribute
|
| 244 |
+
if (
|
| 245 |
+
isinstance(choice, Choices)
|
| 246 |
+
and choice.message
|
| 247 |
+
and choice.message.content
|
| 248 |
+
):
|
| 249 |
+
(
|
| 250 |
+
reasoning_content,
|
| 251 |
+
main_content,
|
| 252 |
+
) = self._extract_reasoning_from_content(choice.message.content)
|
| 253 |
+
|
| 254 |
+
if reasoning_content:
|
| 255 |
+
# Set the reasoning_content field
|
| 256 |
+
choice.message.reasoning_content = reasoning_content
|
| 257 |
+
# Update the main content without reasoning tags
|
| 258 |
+
choice.message.content = main_content
|
| 259 |
+
|
| 260 |
+
return model_response
|
| 261 |
+
|
| 262 |
+
def get_error_class(
|
| 263 |
+
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
| 264 |
+
) -> BedrockError:
|
| 265 |
+
"""Return the appropriate error class for Bedrock."""
|
| 266 |
+
return BedrockError(status_code=status_code, message=error_message)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Handles transforming requests for `bedrock/invoke/{nova} models`
|
| 3 |
+
|
| 4 |
+
Inherits from `AmazonConverseConfig`
|
| 5 |
+
|
| 6 |
+
Nova + Invoke API Tutorial: https://docs.aws.amazon.com/nova/latest/userguide/using-invoke-api.html
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from typing import Any, List, Optional
|
| 10 |
+
|
| 11 |
+
import httpx
|
| 12 |
+
|
| 13 |
+
from litellm.litellm_core_utils.litellm_logging import Logging
|
| 14 |
+
from litellm.types.llms.bedrock import BedrockInvokeNovaRequest
|
| 15 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 16 |
+
from litellm.types.utils import ModelResponse
|
| 17 |
+
|
| 18 |
+
from ..converse_transformation import AmazonConverseConfig
|
| 19 |
+
from .base_invoke_transformation import AmazonInvokeConfig
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
|
| 23 |
+
"""
|
| 24 |
+
Config for sending `nova` requests to `/bedrock/invoke/`
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
def __init__(self, **kwargs):
|
| 28 |
+
super().__init__(**kwargs)
|
| 29 |
+
|
| 30 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 31 |
+
return AmazonConverseConfig.get_supported_openai_params(self, model)
|
| 32 |
+
|
| 33 |
+
def map_openai_params(
|
| 34 |
+
self,
|
| 35 |
+
non_default_params: dict,
|
| 36 |
+
optional_params: dict,
|
| 37 |
+
model: str,
|
| 38 |
+
drop_params: bool,
|
| 39 |
+
) -> dict:
|
| 40 |
+
return AmazonConverseConfig.map_openai_params(
|
| 41 |
+
self, non_default_params, optional_params, model, drop_params
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
def transform_request(
|
| 45 |
+
self,
|
| 46 |
+
model: str,
|
| 47 |
+
messages: List[AllMessageValues],
|
| 48 |
+
optional_params: dict,
|
| 49 |
+
litellm_params: dict,
|
| 50 |
+
headers: dict,
|
| 51 |
+
) -> dict:
|
| 52 |
+
_transformed_nova_request = AmazonConverseConfig.transform_request(
|
| 53 |
+
self,
|
| 54 |
+
model=model,
|
| 55 |
+
messages=messages,
|
| 56 |
+
optional_params=optional_params,
|
| 57 |
+
litellm_params=litellm_params,
|
| 58 |
+
headers=headers,
|
| 59 |
+
)
|
| 60 |
+
_bedrock_invoke_nova_request = BedrockInvokeNovaRequest(
|
| 61 |
+
**_transformed_nova_request
|
| 62 |
+
)
|
| 63 |
+
self._remove_empty_system_messages(_bedrock_invoke_nova_request)
|
| 64 |
+
bedrock_invoke_nova_request = self._filter_allowed_fields(
|
| 65 |
+
_bedrock_invoke_nova_request
|
| 66 |
+
)
|
| 67 |
+
return bedrock_invoke_nova_request
|
| 68 |
+
|
| 69 |
+
def transform_response(
|
| 70 |
+
self,
|
| 71 |
+
model: str,
|
| 72 |
+
raw_response: httpx.Response,
|
| 73 |
+
model_response: ModelResponse,
|
| 74 |
+
logging_obj: Logging,
|
| 75 |
+
request_data: dict,
|
| 76 |
+
messages: List[AllMessageValues],
|
| 77 |
+
optional_params: dict,
|
| 78 |
+
litellm_params: dict,
|
| 79 |
+
encoding: Any,
|
| 80 |
+
api_key: Optional[str] = None,
|
| 81 |
+
json_mode: Optional[bool] = None,
|
| 82 |
+
) -> ModelResponse:
|
| 83 |
+
return AmazonConverseConfig.transform_response(
|
| 84 |
+
self,
|
| 85 |
+
model,
|
| 86 |
+
raw_response,
|
| 87 |
+
model_response,
|
| 88 |
+
logging_obj,
|
| 89 |
+
request_data,
|
| 90 |
+
messages,
|
| 91 |
+
optional_params,
|
| 92 |
+
litellm_params,
|
| 93 |
+
encoding,
|
| 94 |
+
api_key,
|
| 95 |
+
json_mode,
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
def _filter_allowed_fields(
|
| 99 |
+
self, bedrock_invoke_nova_request: BedrockInvokeNovaRequest
|
| 100 |
+
) -> dict:
|
| 101 |
+
"""
|
| 102 |
+
Filter out fields that are not allowed in the `BedrockInvokeNovaRequest` dataclass.
|
| 103 |
+
"""
|
| 104 |
+
allowed_fields = set(BedrockInvokeNovaRequest.__annotations__.keys())
|
| 105 |
+
return {
|
| 106 |
+
k: v for k, v in bedrock_invoke_nova_request.items() if k in allowed_fields
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
def _remove_empty_system_messages(
|
| 110 |
+
self, bedrock_invoke_nova_request: BedrockInvokeNovaRequest
|
| 111 |
+
) -> None:
|
| 112 |
+
"""
|
| 113 |
+
In-place remove empty `system` messages from the request.
|
| 114 |
+
|
| 115 |
+
/bedrock/invoke/ does not allow empty `system` messages.
|
| 116 |
+
"""
|
| 117 |
+
_system_message = bedrock_invoke_nova_request.get("system", None)
|
| 118 |
+
if isinstance(_system_message, list) and len(_system_message) == 0:
|
| 119 |
+
bedrock_invoke_nova_request.pop("system", None)
|
| 120 |
+
return
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation for Bedrock imported models that use OpenAI Chat Completions format.
|
| 3 |
+
|
| 4 |
+
Use this for models imported into Bedrock that accept the OpenAI API format.
|
| 5 |
+
Model format: bedrock/openai/<model-id>
|
| 6 |
+
|
| 7 |
+
Example: bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union
|
| 11 |
+
|
| 12 |
+
import httpx
|
| 13 |
+
|
| 14 |
+
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
| 15 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 16 |
+
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 17 |
+
from litellm.passthrough.utils import CommonUtils
|
| 18 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 19 |
+
|
| 20 |
+
if TYPE_CHECKING:
|
| 21 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 22 |
+
|
| 23 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 24 |
+
else:
|
| 25 |
+
LiteLLMLoggingObj = Any
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM):
|
| 29 |
+
"""
|
| 30 |
+
Configuration for Bedrock imported models that use OpenAI Chat Completions format.
|
| 31 |
+
|
| 32 |
+
This class handles the transformation of requests and responses for Bedrock
|
| 33 |
+
imported models that accept the OpenAI API format directly.
|
| 34 |
+
|
| 35 |
+
Inherits from OpenAIGPTConfig to leverage standard OpenAI parameter handling
|
| 36 |
+
and response transformation, while adding Bedrock-specific URL generation
|
| 37 |
+
and AWS request signing.
|
| 38 |
+
|
| 39 |
+
Usage:
|
| 40 |
+
model = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123"
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
def __init__(self, **kwargs):
|
| 44 |
+
OpenAIGPTConfig.__init__(self, **kwargs)
|
| 45 |
+
BaseAWSLLM.__init__(self, **kwargs)
|
| 46 |
+
|
| 47 |
+
@property
|
| 48 |
+
def custom_llm_provider(self) -> Optional[str]:
|
| 49 |
+
return "bedrock"
|
| 50 |
+
|
| 51 |
+
def _get_openai_model_id(self, model: str) -> str:
|
| 52 |
+
"""
|
| 53 |
+
Extract the actual model ID from the LiteLLM model name.
|
| 54 |
+
|
| 55 |
+
Input format: bedrock/openai/<model-id>
|
| 56 |
+
Returns: <model-id>
|
| 57 |
+
"""
|
| 58 |
+
# Remove bedrock/ prefix if present
|
| 59 |
+
if model.startswith("bedrock/"):
|
| 60 |
+
model = model[8:]
|
| 61 |
+
|
| 62 |
+
# Remove openai/ prefix
|
| 63 |
+
if model.startswith("openai/"):
|
| 64 |
+
model = model[7:]
|
| 65 |
+
|
| 66 |
+
return model
|
| 67 |
+
|
| 68 |
+
def get_complete_url(
|
| 69 |
+
self,
|
| 70 |
+
api_base: Optional[str],
|
| 71 |
+
api_key: Optional[str],
|
| 72 |
+
model: str,
|
| 73 |
+
optional_params: dict,
|
| 74 |
+
litellm_params: dict,
|
| 75 |
+
stream: Optional[bool] = None,
|
| 76 |
+
) -> str:
|
| 77 |
+
"""
|
| 78 |
+
Get the complete URL for the Bedrock invoke endpoint.
|
| 79 |
+
|
| 80 |
+
Uses the standard Bedrock invoke endpoint format.
|
| 81 |
+
"""
|
| 82 |
+
model_id = self._get_openai_model_id(model)
|
| 83 |
+
|
| 84 |
+
# Get AWS region
|
| 85 |
+
aws_region_name = self._get_aws_region_name(
|
| 86 |
+
optional_params=optional_params, model=model
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
# Get runtime endpoint
|
| 90 |
+
aws_bedrock_runtime_endpoint = optional_params.get(
|
| 91 |
+
"aws_bedrock_runtime_endpoint", None
|
| 92 |
+
)
|
| 93 |
+
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
|
| 94 |
+
api_base=api_base,
|
| 95 |
+
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
| 96 |
+
aws_region_name=aws_region_name,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
# Encode model ID for ARNs (e.g., :imported-model/ -> :imported-model%2F)
|
| 100 |
+
model_id = CommonUtils.encode_bedrock_runtime_modelid_arn(model_id)
|
| 101 |
+
|
| 102 |
+
# Build the invoke URL
|
| 103 |
+
if stream:
|
| 104 |
+
endpoint_url = (
|
| 105 |
+
f"{endpoint_url}/model/{model_id}/invoke-with-response-stream"
|
| 106 |
+
)
|
| 107 |
+
else:
|
| 108 |
+
endpoint_url = f"{endpoint_url}/model/{model_id}/invoke"
|
| 109 |
+
|
| 110 |
+
return endpoint_url
|
| 111 |
+
|
| 112 |
+
def sign_request(
|
| 113 |
+
self,
|
| 114 |
+
headers: dict,
|
| 115 |
+
optional_params: dict,
|
| 116 |
+
request_data: dict,
|
| 117 |
+
api_base: str,
|
| 118 |
+
api_key: Optional[str] = None,
|
| 119 |
+
model: Optional[str] = None,
|
| 120 |
+
stream: Optional[bool] = None,
|
| 121 |
+
fake_stream: Optional[bool] = None,
|
| 122 |
+
) -> Tuple[dict, Optional[bytes]]:
|
| 123 |
+
"""
|
| 124 |
+
Sign the request using AWS Signature Version 4.
|
| 125 |
+
"""
|
| 126 |
+
return self._sign_request(
|
| 127 |
+
service_name="bedrock",
|
| 128 |
+
headers=headers,
|
| 129 |
+
optional_params=optional_params,
|
| 130 |
+
request_data=request_data,
|
| 131 |
+
api_base=api_base,
|
| 132 |
+
api_key=api_key,
|
| 133 |
+
model=model,
|
| 134 |
+
stream=stream,
|
| 135 |
+
fake_stream=fake_stream,
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
def transform_request(
|
| 139 |
+
self,
|
| 140 |
+
model: str,
|
| 141 |
+
messages: List[AllMessageValues],
|
| 142 |
+
optional_params: dict,
|
| 143 |
+
litellm_params: dict,
|
| 144 |
+
headers: dict,
|
| 145 |
+
) -> dict:
|
| 146 |
+
"""
|
| 147 |
+
Transform the request to OpenAI Chat Completions format for Bedrock imported models.
|
| 148 |
+
|
| 149 |
+
Removes AWS-specific params and stream param (handled separately in URL),
|
| 150 |
+
then delegates to parent class for standard OpenAI request transformation.
|
| 151 |
+
"""
|
| 152 |
+
# Remove stream from optional_params as it's handled separately in URL
|
| 153 |
+
optional_params.pop("stream", None)
|
| 154 |
+
|
| 155 |
+
# Remove AWS-specific params that shouldn't be in the request body
|
| 156 |
+
inference_params = {
|
| 157 |
+
k: v
|
| 158 |
+
for k, v in optional_params.items()
|
| 159 |
+
if k not in self.aws_authentication_params
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
# Use parent class transform_request for OpenAI format
|
| 163 |
+
return super().transform_request(
|
| 164 |
+
model=self._get_openai_model_id(model),
|
| 165 |
+
messages=messages,
|
| 166 |
+
optional_params=inference_params,
|
| 167 |
+
litellm_params=litellm_params,
|
| 168 |
+
headers=headers,
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
def validate_environment(
|
| 172 |
+
self,
|
| 173 |
+
headers: dict,
|
| 174 |
+
model: str,
|
| 175 |
+
messages: List[AllMessageValues],
|
| 176 |
+
optional_params: dict,
|
| 177 |
+
litellm_params: dict,
|
| 178 |
+
api_key: Optional[str] = None,
|
| 179 |
+
api_base: Optional[str] = None,
|
| 180 |
+
) -> dict:
|
| 181 |
+
"""
|
| 182 |
+
Validate the environment and return headers.
|
| 183 |
+
|
| 184 |
+
For Bedrock, we don't need Bearer token auth since we use AWS SigV4.
|
| 185 |
+
"""
|
| 186 |
+
return headers
|
| 187 |
+
|
| 188 |
+
def get_error_class(
|
| 189 |
+
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
| 190 |
+
) -> BedrockError:
|
| 191 |
+
"""Return the appropriate error class for Bedrock."""
|
| 192 |
+
return BedrockError(status_code=status_code, message=error_message)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Handles transforming requests for `bedrock/invoke/{qwen2} models`
|
| 3 |
+
|
| 4 |
+
Inherits from `AmazonQwen3Config` since Qwen2 and Qwen3 architectures are mostly similar.
|
| 5 |
+
The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field.
|
| 6 |
+
|
| 7 |
+
Qwen2 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from typing import Any, List, Optional
|
| 11 |
+
|
| 12 |
+
import httpx
|
| 13 |
+
|
| 14 |
+
from litellm.llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import (
|
| 15 |
+
AmazonQwen3Config,
|
| 16 |
+
)
|
| 17 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 18 |
+
LiteLLMLoggingObj,
|
| 19 |
+
)
|
| 20 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 21 |
+
from litellm.types.utils import ModelResponse, Usage
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class AmazonQwen2Config(AmazonQwen3Config):
|
| 25 |
+
"""
|
| 26 |
+
Config for sending `qwen2` requests to `/bedrock/invoke/`
|
| 27 |
+
|
| 28 |
+
Inherits from AmazonQwen3Config since Qwen2 and Qwen3 architectures are mostly similar.
|
| 29 |
+
The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field.
|
| 30 |
+
|
| 31 |
+
Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
def transform_response(
|
| 35 |
+
self,
|
| 36 |
+
model: str,
|
| 37 |
+
raw_response: httpx.Response,
|
| 38 |
+
model_response: ModelResponse,
|
| 39 |
+
logging_obj: LiteLLMLoggingObj,
|
| 40 |
+
request_data: dict,
|
| 41 |
+
messages: List[AllMessageValues],
|
| 42 |
+
optional_params: dict,
|
| 43 |
+
litellm_params: dict,
|
| 44 |
+
encoding: Any,
|
| 45 |
+
api_key: Optional[str] = None,
|
| 46 |
+
json_mode: Optional[bool] = None,
|
| 47 |
+
) -> ModelResponse:
|
| 48 |
+
"""
|
| 49 |
+
Transform Qwen2 Bedrock response to OpenAI format
|
| 50 |
+
|
| 51 |
+
Qwen2 uses "text" field, but we also support "generation" field for compatibility.
|
| 52 |
+
"""
|
| 53 |
+
try:
|
| 54 |
+
if hasattr(raw_response, "json"):
|
| 55 |
+
response_data = raw_response.json()
|
| 56 |
+
else:
|
| 57 |
+
response_data = raw_response
|
| 58 |
+
|
| 59 |
+
# Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility
|
| 60 |
+
generated_text = response_data.get("generation", "") or response_data.get(
|
| 61 |
+
"text", ""
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
# Clean up the response (remove assistant start token if present)
|
| 65 |
+
if generated_text.startswith("<|im_start|>assistant\n"):
|
| 66 |
+
generated_text = generated_text[len("<|im_start|>assistant\n") :]
|
| 67 |
+
if generated_text.endswith("<|im_end|>"):
|
| 68 |
+
generated_text = generated_text[: -len("<|im_end|>")]
|
| 69 |
+
|
| 70 |
+
# Set the content in the existing model_response structure
|
| 71 |
+
if hasattr(model_response, "choices") and len(model_response.choices) > 0:
|
| 72 |
+
choice = model_response.choices[0]
|
| 73 |
+
choice.message.content = generated_text
|
| 74 |
+
choice.finish_reason = "stop"
|
| 75 |
+
|
| 76 |
+
# Set usage information if available in response
|
| 77 |
+
if "usage" in response_data:
|
| 78 |
+
usage_data = response_data["usage"]
|
| 79 |
+
setattr(
|
| 80 |
+
model_response,
|
| 81 |
+
"usage",
|
| 82 |
+
Usage(
|
| 83 |
+
prompt_tokens=usage_data.get("prompt_tokens", 0),
|
| 84 |
+
completion_tokens=usage_data.get("completion_tokens", 0),
|
| 85 |
+
total_tokens=usage_data.get("total_tokens", 0),
|
| 86 |
+
),
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
return model_response
|
| 90 |
+
|
| 91 |
+
except Exception as e:
|
| 92 |
+
if logging_obj:
|
| 93 |
+
logging_obj.post_call(
|
| 94 |
+
input=messages,
|
| 95 |
+
api_key=api_key,
|
| 96 |
+
original_response=raw_response,
|
| 97 |
+
additional_args={"error": str(e)},
|
| 98 |
+
)
|
| 99 |
+
raise e
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Handles transforming requests for `bedrock/invoke/{qwen3} models`
|
| 3 |
+
|
| 4 |
+
Inherits from `AmazonInvokeConfig`
|
| 5 |
+
|
| 6 |
+
Qwen3 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from typing import Any, List, Optional
|
| 10 |
+
|
| 11 |
+
import httpx
|
| 12 |
+
|
| 13 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
| 14 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 15 |
+
AmazonInvokeConfig,
|
| 16 |
+
LiteLLMLoggingObj,
|
| 17 |
+
)
|
| 18 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 19 |
+
from litellm.types.utils import ModelResponse, Usage
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
|
| 23 |
+
"""
|
| 24 |
+
Config for sending `qwen3` requests to `/bedrock/invoke/`
|
| 25 |
+
|
| 26 |
+
Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
max_tokens: Optional[int] = None
|
| 30 |
+
temperature: Optional[float] = None
|
| 31 |
+
top_p: Optional[float] = None
|
| 32 |
+
top_k: Optional[int] = None
|
| 33 |
+
stop: Optional[List[str]] = None
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
max_tokens: Optional[int] = None,
|
| 38 |
+
temperature: Optional[float] = None,
|
| 39 |
+
top_p: Optional[float] = None,
|
| 40 |
+
top_k: Optional[int] = None,
|
| 41 |
+
stop: Optional[List[str]] = None,
|
| 42 |
+
) -> None:
|
| 43 |
+
locals_ = locals().copy()
|
| 44 |
+
for key, value in locals_.items():
|
| 45 |
+
if key != "self" and value is not None:
|
| 46 |
+
setattr(self.__class__, key, value)
|
| 47 |
+
AmazonInvokeConfig.__init__(self)
|
| 48 |
+
|
| 49 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 50 |
+
return [
|
| 51 |
+
"max_tokens",
|
| 52 |
+
"temperature",
|
| 53 |
+
"top_p",
|
| 54 |
+
"top_k",
|
| 55 |
+
"stop",
|
| 56 |
+
"stream",
|
| 57 |
+
]
|
| 58 |
+
|
| 59 |
+
def map_openai_params(
|
| 60 |
+
self,
|
| 61 |
+
non_default_params: dict,
|
| 62 |
+
optional_params: dict,
|
| 63 |
+
model: str,
|
| 64 |
+
drop_params: bool,
|
| 65 |
+
) -> dict:
|
| 66 |
+
for k, v in non_default_params.items():
|
| 67 |
+
if k == "max_tokens":
|
| 68 |
+
optional_params["max_tokens"] = v
|
| 69 |
+
if k == "temperature":
|
| 70 |
+
optional_params["temperature"] = v
|
| 71 |
+
if k == "top_p":
|
| 72 |
+
optional_params["top_p"] = v
|
| 73 |
+
if k == "top_k":
|
| 74 |
+
optional_params["top_k"] = v
|
| 75 |
+
if k == "stop":
|
| 76 |
+
optional_params["stop"] = v
|
| 77 |
+
if k == "stream":
|
| 78 |
+
optional_params["stream"] = v
|
| 79 |
+
return optional_params
|
| 80 |
+
|
| 81 |
+
def transform_request(
|
| 82 |
+
self,
|
| 83 |
+
model: str,
|
| 84 |
+
messages: List[AllMessageValues],
|
| 85 |
+
optional_params: dict,
|
| 86 |
+
litellm_params: dict,
|
| 87 |
+
headers: dict,
|
| 88 |
+
) -> dict:
|
| 89 |
+
"""
|
| 90 |
+
Transform OpenAI format to Qwen3 Bedrock invoke format
|
| 91 |
+
"""
|
| 92 |
+
# Convert messages to prompt format
|
| 93 |
+
prompt = self._convert_messages_to_prompt(messages)
|
| 94 |
+
|
| 95 |
+
# Build the request body
|
| 96 |
+
request_body = {
|
| 97 |
+
"prompt": prompt,
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
# Add optional parameters
|
| 101 |
+
if "max_tokens" in optional_params:
|
| 102 |
+
request_body["max_gen_len"] = optional_params["max_tokens"]
|
| 103 |
+
if "temperature" in optional_params:
|
| 104 |
+
request_body["temperature"] = optional_params["temperature"]
|
| 105 |
+
if "top_p" in optional_params:
|
| 106 |
+
request_body["top_p"] = optional_params["top_p"]
|
| 107 |
+
if "top_k" in optional_params:
|
| 108 |
+
request_body["top_k"] = optional_params["top_k"]
|
| 109 |
+
if "stop" in optional_params:
|
| 110 |
+
request_body["stop"] = optional_params["stop"]
|
| 111 |
+
|
| 112 |
+
return request_body
|
| 113 |
+
|
| 114 |
+
def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str:
|
| 115 |
+
"""
|
| 116 |
+
Convert OpenAI messages format to Qwen3 prompt format
|
| 117 |
+
Supports tool calls, multimodal content, and various message types
|
| 118 |
+
"""
|
| 119 |
+
prompt_parts = []
|
| 120 |
+
|
| 121 |
+
for message in messages:
|
| 122 |
+
role = message.get("role", "")
|
| 123 |
+
content = message.get("content", "")
|
| 124 |
+
tool_calls = message.get("tool_calls", [])
|
| 125 |
+
|
| 126 |
+
if role == "system":
|
| 127 |
+
prompt_parts.append(f"<|im_start|>system\n{content}<|im_end|>")
|
| 128 |
+
elif role == "user":
|
| 129 |
+
# Handle multimodal content
|
| 130 |
+
if isinstance(content, list):
|
| 131 |
+
text_content = []
|
| 132 |
+
for item in content:
|
| 133 |
+
if item.get("type") == "text":
|
| 134 |
+
text_content.append(item.get("text", ""))
|
| 135 |
+
elif item.get("type") == "image_url":
|
| 136 |
+
# For Qwen3, we can include image placeholders
|
| 137 |
+
text_content.append(
|
| 138 |
+
"<|vision_start|><|image_pad|><|vision_end|>"
|
| 139 |
+
)
|
| 140 |
+
content = "".join(text_content)
|
| 141 |
+
prompt_parts.append(f"<|im_start|>user\n{content}<|im_end|>")
|
| 142 |
+
elif role == "assistant":
|
| 143 |
+
if tool_calls and isinstance(tool_calls, list):
|
| 144 |
+
# Handle tool calls
|
| 145 |
+
for tool_call in tool_calls:
|
| 146 |
+
function_name = tool_call.get("function", {}).get("name", "")
|
| 147 |
+
function_args = tool_call.get("function", {}).get(
|
| 148 |
+
"arguments", ""
|
| 149 |
+
)
|
| 150 |
+
prompt_parts.append(
|
| 151 |
+
f'<|im_start|>assistant\n<tool_call>\n{{"name": "{function_name}", "arguments": "{function_args}"}}\n</tool_call><|im_end|>'
|
| 152 |
+
)
|
| 153 |
+
else:
|
| 154 |
+
prompt_parts.append(f"<|im_start|>assistant\n{content}<|im_end|>")
|
| 155 |
+
elif role == "tool":
|
| 156 |
+
# Handle tool responses
|
| 157 |
+
prompt_parts.append(f"<|im_start|>tool\n{content}<|im_end|>")
|
| 158 |
+
|
| 159 |
+
# Add assistant start token for response generation
|
| 160 |
+
prompt_parts.append("<|im_start|>assistant\n")
|
| 161 |
+
|
| 162 |
+
return "\n".join(prompt_parts)
|
| 163 |
+
|
| 164 |
+
def transform_response(
|
| 165 |
+
self,
|
| 166 |
+
model: str,
|
| 167 |
+
raw_response: httpx.Response,
|
| 168 |
+
model_response: ModelResponse,
|
| 169 |
+
logging_obj: LiteLLMLoggingObj,
|
| 170 |
+
request_data: dict,
|
| 171 |
+
messages: List[AllMessageValues],
|
| 172 |
+
optional_params: dict,
|
| 173 |
+
litellm_params: dict,
|
| 174 |
+
encoding: Any,
|
| 175 |
+
api_key: Optional[str] = None,
|
| 176 |
+
json_mode: Optional[bool] = None,
|
| 177 |
+
) -> ModelResponse:
|
| 178 |
+
"""
|
| 179 |
+
Transform Qwen3 Bedrock response to OpenAI format
|
| 180 |
+
"""
|
| 181 |
+
try:
|
| 182 |
+
if hasattr(raw_response, "json"):
|
| 183 |
+
response_data = raw_response.json()
|
| 184 |
+
else:
|
| 185 |
+
response_data = raw_response
|
| 186 |
+
|
| 187 |
+
# Extract the generated text - Qwen3 uses "generation" field
|
| 188 |
+
generated_text = response_data.get("generation", "")
|
| 189 |
+
|
| 190 |
+
# Clean up the response (remove assistant start token if present)
|
| 191 |
+
if generated_text.startswith("<|im_start|>assistant\n"):
|
| 192 |
+
generated_text = generated_text[len("<|im_start|>assistant\n") :]
|
| 193 |
+
if generated_text.endswith("<|im_end|>"):
|
| 194 |
+
generated_text = generated_text[: -len("<|im_end|>")]
|
| 195 |
+
|
| 196 |
+
# Set the content in the existing model_response structure
|
| 197 |
+
if hasattr(model_response, "choices") and len(model_response.choices) > 0:
|
| 198 |
+
choice = model_response.choices[0]
|
| 199 |
+
choice.message.content = generated_text
|
| 200 |
+
choice.finish_reason = "stop"
|
| 201 |
+
|
| 202 |
+
# Set usage information if available in response
|
| 203 |
+
if "usage" in response_data:
|
| 204 |
+
usage_data = response_data["usage"]
|
| 205 |
+
setattr(
|
| 206 |
+
model_response,
|
| 207 |
+
"usage",
|
| 208 |
+
Usage(
|
| 209 |
+
prompt_tokens=usage_data.get("prompt_tokens", 0),
|
| 210 |
+
completion_tokens=usage_data.get("completion_tokens", 0),
|
| 211 |
+
total_tokens=usage_data.get("total_tokens", 0),
|
| 212 |
+
),
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
return model_response
|
| 216 |
+
|
| 217 |
+
except Exception as e:
|
| 218 |
+
if logging_obj:
|
| 219 |
+
logging_obj.post_call(
|
| 220 |
+
input=messages,
|
| 221 |
+
api_key=api_key,
|
| 222 |
+
original_response=raw_response,
|
| 223 |
+
additional_args={"error": str(e)},
|
| 224 |
+
)
|
| 225 |
+
raise e
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
import types
|
| 3 |
+
from typing import List, Optional, Union
|
| 4 |
+
|
| 5 |
+
import litellm
|
| 6 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
| 7 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 8 |
+
AmazonInvokeConfig,
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AmazonTitanConfig(AmazonInvokeConfig, BaseConfig):
|
| 13 |
+
"""
|
| 14 |
+
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=titan-text-express-v1
|
| 15 |
+
|
| 16 |
+
Supported Params for the Amazon Titan models:
|
| 17 |
+
|
| 18 |
+
- `maxTokenCount` (integer) max tokens,
|
| 19 |
+
- `stopSequences` (string[]) list of stop sequence strings
|
| 20 |
+
- `temperature` (float) temperature for model,
|
| 21 |
+
- `topP` (int) top p for model
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
maxTokenCount: Optional[int] = None
|
| 25 |
+
stopSequences: Optional[list] = None
|
| 26 |
+
temperature: Optional[float] = None
|
| 27 |
+
topP: Optional[int] = None
|
| 28 |
+
|
| 29 |
+
def __init__(
|
| 30 |
+
self,
|
| 31 |
+
maxTokenCount: Optional[int] = None,
|
| 32 |
+
stopSequences: Optional[list] = None,
|
| 33 |
+
temperature: Optional[float] = None,
|
| 34 |
+
topP: Optional[int] = None,
|
| 35 |
+
) -> None:
|
| 36 |
+
locals_ = locals().copy()
|
| 37 |
+
for key, value in locals_.items():
|
| 38 |
+
if key != "self" and value is not None:
|
| 39 |
+
setattr(self.__class__, key, value)
|
| 40 |
+
|
| 41 |
+
AmazonInvokeConfig.__init__(self)
|
| 42 |
+
|
| 43 |
+
@classmethod
|
| 44 |
+
def get_config(cls):
|
| 45 |
+
return {
|
| 46 |
+
k: v
|
| 47 |
+
for k, v in cls.__dict__.items()
|
| 48 |
+
if not k.startswith("__")
|
| 49 |
+
and not k.startswith("_abc")
|
| 50 |
+
and not isinstance(
|
| 51 |
+
v,
|
| 52 |
+
(
|
| 53 |
+
types.FunctionType,
|
| 54 |
+
types.BuiltinFunctionType,
|
| 55 |
+
classmethod,
|
| 56 |
+
staticmethod,
|
| 57 |
+
),
|
| 58 |
+
)
|
| 59 |
+
and v is not None
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
def _map_and_modify_arg(
|
| 63 |
+
self,
|
| 64 |
+
supported_params: dict,
|
| 65 |
+
provider: str,
|
| 66 |
+
model: str,
|
| 67 |
+
stop: Union[List[str], str],
|
| 68 |
+
):
|
| 69 |
+
"""
|
| 70 |
+
filter params to fit the required provider format, drop those that don't fit if user sets `litellm.drop_params = True`.
|
| 71 |
+
"""
|
| 72 |
+
filtered_stop = None
|
| 73 |
+
if "stop" in supported_params and litellm.drop_params:
|
| 74 |
+
if provider == "bedrock" and "amazon" in model:
|
| 75 |
+
filtered_stop = []
|
| 76 |
+
if isinstance(stop, list):
|
| 77 |
+
for s in stop:
|
| 78 |
+
if re.match(r"^(\|+|User:)$", s):
|
| 79 |
+
filtered_stop.append(s)
|
| 80 |
+
if filtered_stop is not None:
|
| 81 |
+
supported_params["stop"] = filtered_stop
|
| 82 |
+
|
| 83 |
+
return supported_params
|
| 84 |
+
|
| 85 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 86 |
+
return [
|
| 87 |
+
"max_tokens",
|
| 88 |
+
"max_completion_tokens",
|
| 89 |
+
"stop",
|
| 90 |
+
"temperature",
|
| 91 |
+
"top_p",
|
| 92 |
+
"stream",
|
| 93 |
+
]
|
| 94 |
+
|
| 95 |
+
def map_openai_params(
|
| 96 |
+
self,
|
| 97 |
+
non_default_params: dict,
|
| 98 |
+
optional_params: dict,
|
| 99 |
+
model: str,
|
| 100 |
+
drop_params: bool,
|
| 101 |
+
) -> dict:
|
| 102 |
+
for k, v in non_default_params.items():
|
| 103 |
+
if k == "max_tokens" or k == "max_completion_tokens":
|
| 104 |
+
optional_params["maxTokenCount"] = v
|
| 105 |
+
if k == "temperature":
|
| 106 |
+
optional_params["temperature"] = v
|
| 107 |
+
if k == "stop":
|
| 108 |
+
filtered_stop = self._map_and_modify_arg(
|
| 109 |
+
{"stop": v}, provider="bedrock", model=model, stop=v
|
| 110 |
+
)
|
| 111 |
+
optional_params["stopSequences"] = filtered_stop["stop"]
|
| 112 |
+
if k == "top_p":
|
| 113 |
+
optional_params["topP"] = v
|
| 114 |
+
if k == "stream":
|
| 115 |
+
optional_params["stream"] = v
|
| 116 |
+
return optional_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transforms OpenAI-style requests into TwelveLabs Pegasus 1.2 requests for Bedrock.
|
| 3 |
+
|
| 4 |
+
Reference:
|
| 5 |
+
https://docs.twelvelabs.io/docs/models/pegasus
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
import time
|
| 10 |
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
| 11 |
+
|
| 12 |
+
import httpx
|
| 13 |
+
|
| 14 |
+
import litellm
|
| 15 |
+
from litellm._logging import verbose_logger
|
| 16 |
+
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
| 17 |
+
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
| 18 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
| 19 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 20 |
+
AmazonInvokeConfig,
|
| 21 |
+
)
|
| 22 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 23 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 24 |
+
from litellm.types.utils import ModelResponse, Usage
|
| 25 |
+
from litellm.utils import get_base64_str
|
| 26 |
+
|
| 27 |
+
if TYPE_CHECKING:
|
| 28 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 29 |
+
|
| 30 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 31 |
+
else:
|
| 32 |
+
LiteLLMLoggingObj = Any
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
| 36 |
+
"""
|
| 37 |
+
Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
|
| 38 |
+
`twelvelabs.pegasus-1-2-v1:0`.
|
| 39 |
+
|
| 40 |
+
Pegasus 1.2 requires an `inputPrompt` and a `mediaSource` that either references
|
| 41 |
+
an S3 object or a base64-encoded clip. Optional OpenAI params (temperature,
|
| 42 |
+
response_format, max_tokens) are translated to the TwelveLabs schema.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 46 |
+
return [
|
| 47 |
+
"max_tokens",
|
| 48 |
+
"max_completion_tokens",
|
| 49 |
+
"temperature",
|
| 50 |
+
"response_format",
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
def map_openai_params(
|
| 54 |
+
self,
|
| 55 |
+
non_default_params: dict,
|
| 56 |
+
optional_params: dict,
|
| 57 |
+
model: str,
|
| 58 |
+
drop_params: bool,
|
| 59 |
+
) -> dict:
|
| 60 |
+
for param, value in non_default_params.items():
|
| 61 |
+
if param in {"max_tokens", "max_completion_tokens"}:
|
| 62 |
+
optional_params["maxOutputTokens"] = value
|
| 63 |
+
if param == "temperature":
|
| 64 |
+
optional_params["temperature"] = value
|
| 65 |
+
if param == "response_format":
|
| 66 |
+
optional_params["responseFormat"] = self._normalize_response_format(
|
| 67 |
+
value
|
| 68 |
+
)
|
| 69 |
+
return optional_params
|
| 70 |
+
|
| 71 |
+
def _normalize_response_format(self, value: Any) -> Any:
|
| 72 |
+
"""Normalize response_format to TwelveLabs format.
|
| 73 |
+
|
| 74 |
+
TwelveLabs expects:
|
| 75 |
+
{
|
| 76 |
+
"jsonSchema": {...}
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
But OpenAI format is:
|
| 80 |
+
{
|
| 81 |
+
"type": "json_schema",
|
| 82 |
+
"json_schema": {
|
| 83 |
+
"name": "...",
|
| 84 |
+
"schema": {...}
|
| 85 |
+
}
|
| 86 |
+
}
|
| 87 |
+
"""
|
| 88 |
+
if isinstance(value, dict):
|
| 89 |
+
# If it has json_schema field, extract and transform it
|
| 90 |
+
if "json_schema" in value:
|
| 91 |
+
json_schema = value["json_schema"]
|
| 92 |
+
# Extract the schema if nested
|
| 93 |
+
if isinstance(json_schema, dict) and "schema" in json_schema:
|
| 94 |
+
return {"jsonSchema": json_schema["schema"]}
|
| 95 |
+
# Otherwise use json_schema directly
|
| 96 |
+
return {"jsonSchema": json_schema}
|
| 97 |
+
# If it already has jsonSchema, return as is
|
| 98 |
+
if "jsonSchema" in value:
|
| 99 |
+
return value
|
| 100 |
+
# Otherwise return the dict as is
|
| 101 |
+
return value
|
| 102 |
+
return type_to_response_format_param(response_format=value) or value
|
| 103 |
+
|
| 104 |
+
def transform_request(
|
| 105 |
+
self,
|
| 106 |
+
model: str,
|
| 107 |
+
messages: List[AllMessageValues],
|
| 108 |
+
optional_params: dict,
|
| 109 |
+
litellm_params: dict,
|
| 110 |
+
headers: dict,
|
| 111 |
+
) -> dict:
|
| 112 |
+
input_prompt = self._convert_messages_to_prompt(messages=messages)
|
| 113 |
+
request_data: Dict[str, Any] = {"inputPrompt": input_prompt}
|
| 114 |
+
|
| 115 |
+
media_source = self._build_media_source(optional_params)
|
| 116 |
+
if media_source is not None:
|
| 117 |
+
request_data["mediaSource"] = media_source
|
| 118 |
+
|
| 119 |
+
# Handle temperature and maxOutputTokens
|
| 120 |
+
for key in ("temperature", "maxOutputTokens"):
|
| 121 |
+
if key in optional_params:
|
| 122 |
+
request_data[key] = optional_params.get(key)
|
| 123 |
+
|
| 124 |
+
# Handle responseFormat - transform to TwelveLabs format
|
| 125 |
+
if "responseFormat" in optional_params:
|
| 126 |
+
response_format = optional_params["responseFormat"]
|
| 127 |
+
transformed_format = self._normalize_response_format(response_format)
|
| 128 |
+
if transformed_format:
|
| 129 |
+
request_data["responseFormat"] = transformed_format
|
| 130 |
+
|
| 131 |
+
return request_data
|
| 132 |
+
|
| 133 |
+
def _build_media_source(self, optional_params: dict) -> Optional[dict]:
|
| 134 |
+
direct_source = optional_params.get("mediaSource") or optional_params.get(
|
| 135 |
+
"media_source"
|
| 136 |
+
)
|
| 137 |
+
if isinstance(direct_source, dict):
|
| 138 |
+
return direct_source
|
| 139 |
+
|
| 140 |
+
base64_input = optional_params.get("video_base64") or optional_params.get(
|
| 141 |
+
"base64_string"
|
| 142 |
+
)
|
| 143 |
+
if base64_input:
|
| 144 |
+
return {"base64String": get_base64_str(base64_input)}
|
| 145 |
+
|
| 146 |
+
s3_uri = (
|
| 147 |
+
optional_params.get("video_s3_uri")
|
| 148 |
+
or optional_params.get("s3_uri")
|
| 149 |
+
or optional_params.get("media_source_s3_uri")
|
| 150 |
+
)
|
| 151 |
+
if s3_uri:
|
| 152 |
+
s3_location = {"uri": s3_uri}
|
| 153 |
+
bucket_owner = (
|
| 154 |
+
optional_params.get("video_s3_bucket_owner")
|
| 155 |
+
or optional_params.get("s3_bucket_owner")
|
| 156 |
+
or optional_params.get("media_source_bucket_owner")
|
| 157 |
+
)
|
| 158 |
+
if bucket_owner:
|
| 159 |
+
s3_location["bucketOwner"] = bucket_owner
|
| 160 |
+
return {"s3Location": s3_location}
|
| 161 |
+
return None
|
| 162 |
+
|
| 163 |
+
def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str:
|
| 164 |
+
prompt_parts: List[str] = []
|
| 165 |
+
for message in messages:
|
| 166 |
+
role = message.get("role", "user")
|
| 167 |
+
content = message.get("content", "")
|
| 168 |
+
if isinstance(content, list):
|
| 169 |
+
text_fragments = []
|
| 170 |
+
for item in content:
|
| 171 |
+
if isinstance(item, dict):
|
| 172 |
+
item_type = item.get("type")
|
| 173 |
+
if item_type == "text":
|
| 174 |
+
text_fragments.append(item.get("text", ""))
|
| 175 |
+
elif item_type == "image_url":
|
| 176 |
+
text_fragments.append("<image>")
|
| 177 |
+
elif item_type == "video_url":
|
| 178 |
+
text_fragments.append("<video>")
|
| 179 |
+
elif item_type == "audio_url":
|
| 180 |
+
text_fragments.append("<audio>")
|
| 181 |
+
elif isinstance(item, str):
|
| 182 |
+
text_fragments.append(item)
|
| 183 |
+
content = " ".join(text_fragments)
|
| 184 |
+
prompt_parts.append(f"{role}: {content}")
|
| 185 |
+
return "\n".join(part for part in prompt_parts if part).strip()
|
| 186 |
+
|
| 187 |
+
def transform_response(
|
| 188 |
+
self,
|
| 189 |
+
model: str,
|
| 190 |
+
raw_response: httpx.Response,
|
| 191 |
+
model_response: ModelResponse,
|
| 192 |
+
logging_obj: LiteLLMLoggingObj,
|
| 193 |
+
request_data: dict,
|
| 194 |
+
messages: List[AllMessageValues],
|
| 195 |
+
optional_params: dict,
|
| 196 |
+
litellm_params: dict,
|
| 197 |
+
encoding: Any,
|
| 198 |
+
api_key: Optional[str] = None,
|
| 199 |
+
json_mode: Optional[bool] = None,
|
| 200 |
+
) -> ModelResponse:
|
| 201 |
+
"""
|
| 202 |
+
Transform TwelveLabs Pegasus response to LiteLLM format.
|
| 203 |
+
|
| 204 |
+
TwelveLabs response format:
|
| 205 |
+
{
|
| 206 |
+
"message": "...",
|
| 207 |
+
"finishReason": "stop" | "length"
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
LiteLLM format:
|
| 211 |
+
ModelResponse with choices[0].message.content and finish_reason
|
| 212 |
+
"""
|
| 213 |
+
try:
|
| 214 |
+
completion_response = raw_response.json()
|
| 215 |
+
except Exception as e:
|
| 216 |
+
raise BedrockError(
|
| 217 |
+
message=f"Error parsing response: {raw_response.text}, error: {str(e)}",
|
| 218 |
+
status_code=raw_response.status_code,
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
verbose_logger.debug(
|
| 222 |
+
"twelvelabs pegasus response: %s",
|
| 223 |
+
json.dumps(completion_response, indent=4, default=str),
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
# Extract message content
|
| 227 |
+
message_content = completion_response.get("message", "")
|
| 228 |
+
|
| 229 |
+
# Extract finish reason and map to LiteLLM format
|
| 230 |
+
finish_reason_raw = completion_response.get("finishReason", "stop")
|
| 231 |
+
finish_reason = map_finish_reason(finish_reason_raw)
|
| 232 |
+
|
| 233 |
+
# Set the response content
|
| 234 |
+
try:
|
| 235 |
+
if (
|
| 236 |
+
message_content
|
| 237 |
+
and hasattr(model_response.choices[0], "message")
|
| 238 |
+
and getattr(model_response.choices[0].message, "tool_calls", None)
|
| 239 |
+
is None
|
| 240 |
+
):
|
| 241 |
+
model_response.choices[0].message.content = message_content # type: ignore
|
| 242 |
+
model_response.choices[0].finish_reason = finish_reason
|
| 243 |
+
else:
|
| 244 |
+
raise Exception("Unable to set message content")
|
| 245 |
+
except Exception as e:
|
| 246 |
+
raise BedrockError(
|
| 247 |
+
message=f"Error setting response content: {str(e)}. Response: {completion_response}",
|
| 248 |
+
status_code=raw_response.status_code,
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
# Calculate usage from headers
|
| 252 |
+
bedrock_input_tokens = raw_response.headers.get(
|
| 253 |
+
"x-amzn-bedrock-input-token-count", None
|
| 254 |
+
)
|
| 255 |
+
bedrock_output_tokens = raw_response.headers.get(
|
| 256 |
+
"x-amzn-bedrock-output-token-count", None
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
prompt_tokens = int(
|
| 260 |
+
bedrock_input_tokens or litellm.token_counter(messages=messages)
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
completion_tokens = int(
|
| 264 |
+
bedrock_output_tokens
|
| 265 |
+
or litellm.token_counter(
|
| 266 |
+
text=model_response.choices[0].message.content, # type: ignore
|
| 267 |
+
count_response_tokens=True,
|
| 268 |
+
)
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
model_response.created = int(time.time())
|
| 272 |
+
model_response.model = model
|
| 273 |
+
usage = Usage(
|
| 274 |
+
prompt_tokens=prompt_tokens,
|
| 275 |
+
completion_tokens=completion_tokens,
|
| 276 |
+
total_tokens=prompt_tokens + completion_tokens,
|
| 277 |
+
)
|
| 278 |
+
setattr(model_response, "usage", usage)
|
| 279 |
+
|
| 280 |
+
return model_response
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import types
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
import litellm
|
| 5 |
+
|
| 6 |
+
from .base_invoke_transformation import AmazonInvokeConfig
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class AmazonAnthropicConfig(AmazonInvokeConfig):
|
| 10 |
+
"""
|
| 11 |
+
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=claude
|
| 12 |
+
|
| 13 |
+
Supported Params for the Amazon / Anthropic models:
|
| 14 |
+
|
| 15 |
+
- `max_tokens_to_sample` (integer) max tokens,
|
| 16 |
+
- `temperature` (float) model temperature,
|
| 17 |
+
- `top_k` (integer) top k,
|
| 18 |
+
- `top_p` (integer) top p,
|
| 19 |
+
- `stop_sequences` (string[]) list of stop sequences - e.g. ["\\n\\nHuman:"],
|
| 20 |
+
- `anthropic_version` (string) version of anthropic for bedrock - e.g. "bedrock-2023-05-31"
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
max_tokens_to_sample: Optional[int] = litellm.max_tokens
|
| 24 |
+
stop_sequences: Optional[list] = None
|
| 25 |
+
temperature: Optional[float] = None
|
| 26 |
+
top_k: Optional[int] = None
|
| 27 |
+
top_p: Optional[int] = None
|
| 28 |
+
anthropic_version: Optional[str] = None
|
| 29 |
+
|
| 30 |
+
def __init__(
|
| 31 |
+
self,
|
| 32 |
+
max_tokens_to_sample: Optional[int] = None,
|
| 33 |
+
stop_sequences: Optional[list] = None,
|
| 34 |
+
temperature: Optional[float] = None,
|
| 35 |
+
top_k: Optional[int] = None,
|
| 36 |
+
top_p: Optional[int] = None,
|
| 37 |
+
anthropic_version: Optional[str] = None,
|
| 38 |
+
) -> None:
|
| 39 |
+
locals_ = locals().copy()
|
| 40 |
+
for key, value in locals_.items():
|
| 41 |
+
if key != "self" and value is not None:
|
| 42 |
+
setattr(self.__class__, key, value)
|
| 43 |
+
|
| 44 |
+
@classmethod
|
| 45 |
+
def get_config(cls):
|
| 46 |
+
return {
|
| 47 |
+
k: v
|
| 48 |
+
for k, v in cls.__dict__.items()
|
| 49 |
+
if not k.startswith("__")
|
| 50 |
+
and not isinstance(
|
| 51 |
+
v,
|
| 52 |
+
(
|
| 53 |
+
types.FunctionType,
|
| 54 |
+
types.BuiltinFunctionType,
|
| 55 |
+
classmethod,
|
| 56 |
+
staticmethod,
|
| 57 |
+
),
|
| 58 |
+
)
|
| 59 |
+
and v is not None
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
@staticmethod
|
| 63 |
+
def get_legacy_anthropic_model_names():
|
| 64 |
+
return [
|
| 65 |
+
"anthropic.claude-v2",
|
| 66 |
+
"anthropic.claude-instant-v1",
|
| 67 |
+
"anthropic.claude-v2:1",
|
| 68 |
+
]
|
| 69 |
+
|
| 70 |
+
def get_supported_openai_params(self, model: str):
|
| 71 |
+
return [
|
| 72 |
+
"max_tokens",
|
| 73 |
+
"max_completion_tokens",
|
| 74 |
+
"temperature",
|
| 75 |
+
"stop",
|
| 76 |
+
"top_p",
|
| 77 |
+
"stream",
|
| 78 |
+
]
|
| 79 |
+
|
| 80 |
+
def map_openai_params(
|
| 81 |
+
self,
|
| 82 |
+
non_default_params: dict,
|
| 83 |
+
optional_params: dict,
|
| 84 |
+
model: str,
|
| 85 |
+
drop_params: bool,
|
| 86 |
+
):
|
| 87 |
+
for param, value in non_default_params.items():
|
| 88 |
+
if param == "max_tokens" or param == "max_completion_tokens":
|
| 89 |
+
optional_params["max_tokens_to_sample"] = value
|
| 90 |
+
if param == "temperature":
|
| 91 |
+
optional_params["temperature"] = value
|
| 92 |
+
if param == "top_p":
|
| 93 |
+
optional_params["top_p"] = value
|
| 94 |
+
if param == "stop":
|
| 95 |
+
optional_params["stop_sequences"] = value
|
| 96 |
+
if param == "stream" and value is True:
|
| 97 |
+
optional_params["stream"] = value
|
| 98 |
+
return optional_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import TYPE_CHECKING, Any, List, Optional
|
| 2 |
+
|
| 3 |
+
import httpx
|
| 4 |
+
|
| 5 |
+
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
| 6 |
+
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
| 7 |
+
AmazonInvokeConfig,
|
| 8 |
+
)
|
| 9 |
+
from litellm.llms.bedrock.common_utils import (
|
| 10 |
+
get_anthropic_beta_from_headers,
|
| 11 |
+
remove_custom_field_from_tools,
|
| 12 |
+
)
|
| 13 |
+
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
|
| 14 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 15 |
+
from litellm.types.utils import ModelResponse
|
| 16 |
+
|
| 17 |
+
if TYPE_CHECKING:
|
| 18 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 19 |
+
|
| 20 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 21 |
+
else:
|
| 22 |
+
LiteLLMLoggingObj = Any
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
| 26 |
+
"""
|
| 27 |
+
Reference:
|
| 28 |
+
https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=claude
|
| 29 |
+
https://docs.anthropic.com/claude/docs/models-overview#model-comparison
|
| 30 |
+
https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
|
| 31 |
+
|
| 32 |
+
Supported Params for the Amazon / Anthropic Claude models (Claude 3, Claude 4, etc.):
|
| 33 |
+
Supports anthropic_beta parameter for beta features like:
|
| 34 |
+
- computer-use-2025-01-24 (Claude 3.7 Sonnet)
|
| 35 |
+
- computer-use-2024-10-22 (Claude 3.5 Sonnet v2)
|
| 36 |
+
- token-efficient-tools-2025-02-19 (Claude 3.7 Sonnet)
|
| 37 |
+
- interleaved-thinking-2025-05-14 (Claude 4 models)
|
| 38 |
+
- output-128k-2025-02-19 (Claude 3.7 Sonnet)
|
| 39 |
+
- dev-full-thinking-2025-05-14 (Claude 4 models)
|
| 40 |
+
- context-1m-2025-08-07 (Claude Sonnet 4)
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
anthropic_version: str = "bedrock-2023-05-31"
|
| 44 |
+
|
| 45 |
+
@property
|
| 46 |
+
def custom_llm_provider(self) -> Optional[str]:
|
| 47 |
+
return "bedrock"
|
| 48 |
+
|
| 49 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 50 |
+
return AnthropicConfig.get_supported_openai_params(self, model)
|
| 51 |
+
|
| 52 |
+
def map_openai_params(
|
| 53 |
+
self,
|
| 54 |
+
non_default_params: dict,
|
| 55 |
+
optional_params: dict,
|
| 56 |
+
model: str,
|
| 57 |
+
drop_params: bool,
|
| 58 |
+
) -> dict:
|
| 59 |
+
# Force tool-based structured outputs for Bedrock Invoke
|
| 60 |
+
# (similar to VertexAI fix in #19201)
|
| 61 |
+
# Bedrock Invoke doesn't support output_format parameter
|
| 62 |
+
original_model = model
|
| 63 |
+
if "response_format" in non_default_params:
|
| 64 |
+
# Use a model name that forces tool-based approach
|
| 65 |
+
model = "claude-3-sonnet-20240229"
|
| 66 |
+
|
| 67 |
+
optional_params = AnthropicConfig.map_openai_params(
|
| 68 |
+
self,
|
| 69 |
+
non_default_params,
|
| 70 |
+
optional_params,
|
| 71 |
+
model,
|
| 72 |
+
drop_params,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
# Restore original model name
|
| 76 |
+
model = original_model
|
| 77 |
+
|
| 78 |
+
return optional_params
|
| 79 |
+
|
| 80 |
+
def transform_request(
|
| 81 |
+
self,
|
| 82 |
+
model: str,
|
| 83 |
+
messages: List[AllMessageValues],
|
| 84 |
+
optional_params: dict,
|
| 85 |
+
litellm_params: dict,
|
| 86 |
+
headers: dict,
|
| 87 |
+
) -> dict:
|
| 88 |
+
# Filter out AWS authentication parameters before passing to Anthropic transformation
|
| 89 |
+
# AWS params should only be used for signing requests, not included in request body
|
| 90 |
+
filtered_params = {
|
| 91 |
+
k: v
|
| 92 |
+
for k, v in optional_params.items()
|
| 93 |
+
if k not in self.aws_authentication_params
|
| 94 |
+
}
|
| 95 |
+
filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params)
|
| 96 |
+
|
| 97 |
+
_anthropic_request = AnthropicConfig.transform_request(
|
| 98 |
+
self,
|
| 99 |
+
model=model,
|
| 100 |
+
messages=messages,
|
| 101 |
+
optional_params=filtered_params,
|
| 102 |
+
litellm_params=litellm_params,
|
| 103 |
+
headers=headers,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
_anthropic_request.pop("model", None)
|
| 107 |
+
_anthropic_request.pop("stream", None)
|
| 108 |
+
# Bedrock Invoke doesn't support output_format parameter
|
| 109 |
+
_anthropic_request.pop("output_format", None)
|
| 110 |
+
# Bedrock Invoke doesn't support output_config parameter
|
| 111 |
+
# Fixes: https://github.com/BerriAI/litellm/issues/22797
|
| 112 |
+
_anthropic_request.pop("output_config", None)
|
| 113 |
+
if "anthropic_version" not in _anthropic_request:
|
| 114 |
+
_anthropic_request["anthropic_version"] = self.anthropic_version
|
| 115 |
+
|
| 116 |
+
# Remove `custom` field from tools (Bedrock doesn't support it)
|
| 117 |
+
# Claude Code sends `custom: {defer_loading: true}` on tool definitions,
|
| 118 |
+
# which causes Bedrock to reject the request with "Extra inputs are not permitted"
|
| 119 |
+
# Ref: https://github.com/BerriAI/litellm/issues/22847
|
| 120 |
+
remove_custom_field_from_tools(_anthropic_request)
|
| 121 |
+
|
| 122 |
+
tools = optional_params.get("tools")
|
| 123 |
+
tool_search_used = self.is_tool_search_used(tools)
|
| 124 |
+
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools)
|
| 125 |
+
input_examples_used = self.is_input_examples_used(tools)
|
| 126 |
+
|
| 127 |
+
beta_set = set(get_anthropic_beta_from_headers(headers))
|
| 128 |
+
auto_betas = self.get_anthropic_beta_list(
|
| 129 |
+
model=model,
|
| 130 |
+
optional_params=optional_params,
|
| 131 |
+
computer_tool_used=self.is_computer_tool_used(tools),
|
| 132 |
+
prompt_caching_set=False,
|
| 133 |
+
file_id_used=self.is_file_id_used(messages),
|
| 134 |
+
mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
|
| 135 |
+
)
|
| 136 |
+
beta_set.update(auto_betas)
|
| 137 |
+
|
| 138 |
+
if tool_search_used and not (
|
| 139 |
+
programmatic_tool_calling_used or input_examples_used
|
| 140 |
+
):
|
| 141 |
+
beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
|
| 142 |
+
if "opus-4" in model.lower() or "opus_4" in model.lower():
|
| 143 |
+
beta_set.add("tool-search-tool-2025-10-19")
|
| 144 |
+
|
| 145 |
+
# Filter out beta headers that Bedrock Invoke doesn't support
|
| 146 |
+
# Uses centralized configuration from anthropic_beta_headers_config.json
|
| 147 |
+
beta_list = list(beta_set)
|
| 148 |
+
_anthropic_request["anthropic_beta"] = beta_list
|
| 149 |
+
|
| 150 |
+
return _anthropic_request
|
| 151 |
+
|
| 152 |
+
def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict:
|
| 153 |
+
"""
|
| 154 |
+
Convert tool search entries to the format supported by the Bedrock Invoke API.
|
| 155 |
+
"""
|
| 156 |
+
tools = optional_params.get("tools")
|
| 157 |
+
if not tools or not isinstance(tools, list):
|
| 158 |
+
return optional_params
|
| 159 |
+
|
| 160 |
+
normalized_tools = []
|
| 161 |
+
for tool in tools:
|
| 162 |
+
tool_type = tool.get("type")
|
| 163 |
+
if tool_type == "tool_search_tool_bm25_20251119":
|
| 164 |
+
# Bedrock Invoke does not support the BM25 variant, so skip it.
|
| 165 |
+
continue
|
| 166 |
+
if tool_type == "tool_search_tool_regex_20251119":
|
| 167 |
+
normalized_tool = tool.copy()
|
| 168 |
+
normalized_tool["type"] = "tool_search_tool_regex"
|
| 169 |
+
normalized_tool["name"] = normalized_tool.get(
|
| 170 |
+
"name", "tool_search_tool_regex"
|
| 171 |
+
)
|
| 172 |
+
normalized_tools.append(normalized_tool)
|
| 173 |
+
continue
|
| 174 |
+
normalized_tools.append(tool)
|
| 175 |
+
|
| 176 |
+
optional_params["tools"] = normalized_tools
|
| 177 |
+
return optional_params
|
| 178 |
+
|
| 179 |
+
def transform_response(
|
| 180 |
+
self,
|
| 181 |
+
model: str,
|
| 182 |
+
raw_response: httpx.Response,
|
| 183 |
+
model_response: ModelResponse,
|
| 184 |
+
logging_obj: LiteLLMLoggingObj,
|
| 185 |
+
request_data: dict,
|
| 186 |
+
messages: List[AllMessageValues],
|
| 187 |
+
optional_params: dict,
|
| 188 |
+
litellm_params: dict,
|
| 189 |
+
encoding: Any,
|
| 190 |
+
api_key: Optional[str] = None,
|
| 191 |
+
json_mode: Optional[bool] = None,
|
| 192 |
+
) -> ModelResponse:
|
| 193 |
+
return AnthropicConfig.transform_response(
|
| 194 |
+
self,
|
| 195 |
+
model=model,
|
| 196 |
+
raw_response=raw_response,
|
| 197 |
+
model_response=model_response,
|
| 198 |
+
logging_obj=logging_obj,
|
| 199 |
+
request_data=request_data,
|
| 200 |
+
messages=messages,
|
| 201 |
+
optional_params=optional_params,
|
| 202 |
+
litellm_params=litellm_params,
|
| 203 |
+
encoding=encoding,
|
| 204 |
+
api_key=api_key,
|
| 205 |
+
json_mode=json_mode,
|
| 206 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
ADDED
|
@@ -0,0 +1,613 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import json
|
| 3 |
+
import time
|
| 4 |
+
from functools import partial
|
| 5 |
+
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast, get_args
|
| 6 |
+
|
| 7 |
+
import httpx
|
| 8 |
+
|
| 9 |
+
import litellm
|
| 10 |
+
from litellm._logging import verbose_logger
|
| 11 |
+
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
| 12 |
+
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
| 13 |
+
from litellm.litellm_core_utils.prompt_templates.factory import (
|
| 14 |
+
cohere_message_pt,
|
| 15 |
+
custom_prompt,
|
| 16 |
+
deepseek_r1_pt,
|
| 17 |
+
prompt_factory,
|
| 18 |
+
)
|
| 19 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
| 20 |
+
from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call
|
| 21 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 22 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 23 |
+
AsyncHTTPHandler,
|
| 24 |
+
HTTPHandler,
|
| 25 |
+
_get_httpx_client,
|
| 26 |
+
)
|
| 27 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 28 |
+
from litellm.types.utils import ModelResponse, Usage
|
| 29 |
+
from litellm.utils import CustomStreamWrapper
|
| 30 |
+
|
| 31 |
+
if TYPE_CHECKING:
|
| 32 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 33 |
+
|
| 34 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 35 |
+
else:
|
| 36 |
+
LiteLLMLoggingObj = Any
|
| 37 |
+
|
| 38 |
+
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
| 42 |
+
def __init__(self, **kwargs):
|
| 43 |
+
BaseConfig.__init__(self, **kwargs)
|
| 44 |
+
BaseAWSLLM.__init__(self, **kwargs)
|
| 45 |
+
|
| 46 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 47 |
+
"""
|
| 48 |
+
This is a base invoke model mapping. For Invoke - define a bedrock provider specific config that extends this class.
|
| 49 |
+
"""
|
| 50 |
+
return [
|
| 51 |
+
"max_tokens",
|
| 52 |
+
"max_completion_tokens",
|
| 53 |
+
"stream",
|
| 54 |
+
]
|
| 55 |
+
|
| 56 |
+
def map_openai_params(
|
| 57 |
+
self,
|
| 58 |
+
non_default_params: dict,
|
| 59 |
+
optional_params: dict,
|
| 60 |
+
model: str,
|
| 61 |
+
drop_params: bool,
|
| 62 |
+
) -> dict:
|
| 63 |
+
"""
|
| 64 |
+
This is a base invoke model mapping. For Invoke - define a bedrock provider specific config that extends this class.
|
| 65 |
+
"""
|
| 66 |
+
for param, value in non_default_params.items():
|
| 67 |
+
if param == "max_tokens" or param == "max_completion_tokens":
|
| 68 |
+
optional_params["max_tokens"] = value
|
| 69 |
+
if param == "stream":
|
| 70 |
+
optional_params["stream"] = value
|
| 71 |
+
return optional_params
|
| 72 |
+
|
| 73 |
+
def get_complete_url(
|
| 74 |
+
self,
|
| 75 |
+
api_base: Optional[str],
|
| 76 |
+
api_key: Optional[str],
|
| 77 |
+
model: str,
|
| 78 |
+
optional_params: dict,
|
| 79 |
+
litellm_params: dict,
|
| 80 |
+
stream: Optional[bool] = None,
|
| 81 |
+
) -> str:
|
| 82 |
+
"""
|
| 83 |
+
Get the complete url for the request
|
| 84 |
+
"""
|
| 85 |
+
provider = self.get_bedrock_invoke_provider(model)
|
| 86 |
+
modelId = self.get_bedrock_model_id(
|
| 87 |
+
model=model,
|
| 88 |
+
provider=provider,
|
| 89 |
+
optional_params=optional_params,
|
| 90 |
+
)
|
| 91 |
+
### SET RUNTIME ENDPOINT ###
|
| 92 |
+
aws_bedrock_runtime_endpoint = optional_params.get(
|
| 93 |
+
"aws_bedrock_runtime_endpoint", None
|
| 94 |
+
) # https://bedrock-runtime.{region_name}.amazonaws.com
|
| 95 |
+
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
|
| 96 |
+
api_base=api_base,
|
| 97 |
+
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
| 98 |
+
aws_region_name=self._get_aws_region_name(
|
| 99 |
+
optional_params=optional_params, model=model
|
| 100 |
+
),
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
if (stream is not None and stream is True) and provider != "ai21":
|
| 104 |
+
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream"
|
| 105 |
+
proxy_endpoint_url = (
|
| 106 |
+
f"{proxy_endpoint_url}/model/{modelId}/invoke-with-response-stream"
|
| 107 |
+
)
|
| 108 |
+
else:
|
| 109 |
+
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
|
| 110 |
+
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
|
| 111 |
+
|
| 112 |
+
return endpoint_url
|
| 113 |
+
|
| 114 |
+
def sign_request(
|
| 115 |
+
self,
|
| 116 |
+
headers: dict,
|
| 117 |
+
optional_params: dict,
|
| 118 |
+
request_data: dict,
|
| 119 |
+
api_base: str,
|
| 120 |
+
api_key: Optional[str] = None,
|
| 121 |
+
model: Optional[str] = None,
|
| 122 |
+
stream: Optional[bool] = None,
|
| 123 |
+
fake_stream: Optional[bool] = None,
|
| 124 |
+
) -> Tuple[dict, Optional[bytes]]:
|
| 125 |
+
return self._sign_request(
|
| 126 |
+
service_name="bedrock",
|
| 127 |
+
headers=headers,
|
| 128 |
+
optional_params=optional_params,
|
| 129 |
+
request_data=request_data,
|
| 130 |
+
api_base=api_base,
|
| 131 |
+
api_key=api_key,
|
| 132 |
+
model=model,
|
| 133 |
+
stream=stream,
|
| 134 |
+
fake_stream=fake_stream,
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
def _apply_config_to_params(self, config: dict, inference_params: dict) -> None:
|
| 138 |
+
"""Apply config values to inference_params if not already set."""
|
| 139 |
+
for k, v in config.items():
|
| 140 |
+
if k not in inference_params:
|
| 141 |
+
inference_params[k] = v
|
| 142 |
+
|
| 143 |
+
def transform_request(
|
| 144 |
+
self,
|
| 145 |
+
model: str,
|
| 146 |
+
messages: List[AllMessageValues],
|
| 147 |
+
optional_params: dict,
|
| 148 |
+
litellm_params: dict,
|
| 149 |
+
headers: dict,
|
| 150 |
+
) -> dict:
|
| 151 |
+
## SETUP ##
|
| 152 |
+
stream = optional_params.pop("stream", None)
|
| 153 |
+
custom_prompt_dict: dict = litellm_params.pop("custom_prompt_dict", None) or {}
|
| 154 |
+
hf_model_name = litellm_params.get("hf_model_name", None)
|
| 155 |
+
|
| 156 |
+
provider = self.get_bedrock_invoke_provider(model)
|
| 157 |
+
|
| 158 |
+
prompt, chat_history = self.convert_messages_to_prompt(
|
| 159 |
+
model=hf_model_name or model,
|
| 160 |
+
messages=messages,
|
| 161 |
+
provider=provider,
|
| 162 |
+
custom_prompt_dict=custom_prompt_dict,
|
| 163 |
+
)
|
| 164 |
+
inference_params = copy.deepcopy(optional_params)
|
| 165 |
+
inference_params = {
|
| 166 |
+
k: v
|
| 167 |
+
for k, v in inference_params.items()
|
| 168 |
+
if k not in self.aws_authentication_params
|
| 169 |
+
}
|
| 170 |
+
request_data: dict = {}
|
| 171 |
+
if provider == "cohere":
|
| 172 |
+
if model.startswith("cohere.command-r"):
|
| 173 |
+
## LOAD CONFIG
|
| 174 |
+
config = litellm.AmazonCohereChatConfig().get_config()
|
| 175 |
+
self._apply_config_to_params(config, inference_params)
|
| 176 |
+
_data = {"message": prompt, **inference_params}
|
| 177 |
+
if chat_history is not None:
|
| 178 |
+
_data["chat_history"] = chat_history
|
| 179 |
+
request_data = _data
|
| 180 |
+
else:
|
| 181 |
+
## LOAD CONFIG
|
| 182 |
+
config = litellm.AmazonCohereConfig.get_config()
|
| 183 |
+
self._apply_config_to_params(config, inference_params)
|
| 184 |
+
if stream is True:
|
| 185 |
+
inference_params[
|
| 186 |
+
"stream"
|
| 187 |
+
] = True # cohere requires stream = True in inference params
|
| 188 |
+
request_data = {"prompt": prompt, **inference_params}
|
| 189 |
+
elif provider == "anthropic":
|
| 190 |
+
transformed_request = (
|
| 191 |
+
litellm.AmazonAnthropicClaudeConfig().transform_request(
|
| 192 |
+
model=model,
|
| 193 |
+
messages=messages,
|
| 194 |
+
optional_params=optional_params,
|
| 195 |
+
litellm_params=litellm_params,
|
| 196 |
+
headers=headers,
|
| 197 |
+
)
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
return transformed_request
|
| 201 |
+
elif provider == "nova":
|
| 202 |
+
return litellm.AmazonInvokeNovaConfig().transform_request(
|
| 203 |
+
model=model,
|
| 204 |
+
messages=messages,
|
| 205 |
+
optional_params=optional_params,
|
| 206 |
+
litellm_params=litellm_params,
|
| 207 |
+
headers=headers,
|
| 208 |
+
)
|
| 209 |
+
elif provider == "ai21":
|
| 210 |
+
## LOAD CONFIG
|
| 211 |
+
config = litellm.AmazonAI21Config.get_config()
|
| 212 |
+
self._apply_config_to_params(config, inference_params)
|
| 213 |
+
request_data = {"prompt": prompt, **inference_params}
|
| 214 |
+
elif provider == "mistral":
|
| 215 |
+
## LOAD CONFIG
|
| 216 |
+
config = litellm.AmazonMistralConfig.get_config()
|
| 217 |
+
self._apply_config_to_params(config, inference_params)
|
| 218 |
+
request_data = {"prompt": prompt, **inference_params}
|
| 219 |
+
elif provider == "amazon": # amazon titan
|
| 220 |
+
## LOAD CONFIG
|
| 221 |
+
config = litellm.AmazonTitanConfig.get_config()
|
| 222 |
+
self._apply_config_to_params(config, inference_params)
|
| 223 |
+
request_data = {
|
| 224 |
+
"inputText": prompt,
|
| 225 |
+
"textGenerationConfig": inference_params,
|
| 226 |
+
}
|
| 227 |
+
elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
|
| 228 |
+
## LOAD CONFIG
|
| 229 |
+
config = litellm.AmazonLlamaConfig.get_config()
|
| 230 |
+
self._apply_config_to_params(config, inference_params)
|
| 231 |
+
request_data = {"prompt": prompt, **inference_params}
|
| 232 |
+
elif provider == "twelvelabs":
|
| 233 |
+
return litellm.AmazonTwelveLabsPegasusConfig().transform_request(
|
| 234 |
+
model=model,
|
| 235 |
+
messages=messages,
|
| 236 |
+
optional_params=optional_params,
|
| 237 |
+
litellm_params=litellm_params,
|
| 238 |
+
headers=headers,
|
| 239 |
+
)
|
| 240 |
+
elif provider == "openai":
|
| 241 |
+
# OpenAI imported models use OpenAI Chat Completions format
|
| 242 |
+
return litellm.AmazonBedrockOpenAIConfig().transform_request(
|
| 243 |
+
model=model,
|
| 244 |
+
messages=messages,
|
| 245 |
+
optional_params=optional_params,
|
| 246 |
+
litellm_params=litellm_params,
|
| 247 |
+
headers=headers,
|
| 248 |
+
)
|
| 249 |
+
else:
|
| 250 |
+
raise BedrockError(
|
| 251 |
+
status_code=404,
|
| 252 |
+
message="Bedrock Invoke HTTPX: Unknown provider={}, model={}. Try calling via converse route - `bedrock/converse/<model>`.".format(
|
| 253 |
+
provider, model
|
| 254 |
+
),
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
return request_data
|
| 258 |
+
|
| 259 |
+
def transform_response( # noqa: PLR0915
|
| 260 |
+
self,
|
| 261 |
+
model: str,
|
| 262 |
+
raw_response: httpx.Response,
|
| 263 |
+
model_response: ModelResponse,
|
| 264 |
+
logging_obj: LiteLLMLoggingObj,
|
| 265 |
+
request_data: dict,
|
| 266 |
+
messages: List[AllMessageValues],
|
| 267 |
+
optional_params: dict,
|
| 268 |
+
litellm_params: dict,
|
| 269 |
+
encoding: Any,
|
| 270 |
+
api_key: Optional[str] = None,
|
| 271 |
+
json_mode: Optional[bool] = None,
|
| 272 |
+
) -> ModelResponse:
|
| 273 |
+
try:
|
| 274 |
+
completion_response = raw_response.json()
|
| 275 |
+
except Exception:
|
| 276 |
+
raise BedrockError(
|
| 277 |
+
message=raw_response.text, status_code=raw_response.status_code
|
| 278 |
+
)
|
| 279 |
+
verbose_logger.debug(
|
| 280 |
+
"bedrock invoke response % s",
|
| 281 |
+
json.dumps(completion_response, indent=4, default=str),
|
| 282 |
+
)
|
| 283 |
+
provider = self.get_bedrock_invoke_provider(model)
|
| 284 |
+
outputText: Optional[str] = None
|
| 285 |
+
try:
|
| 286 |
+
if provider == "cohere":
|
| 287 |
+
if "text" in completion_response:
|
| 288 |
+
outputText = completion_response["text"] # type: ignore
|
| 289 |
+
elif "generations" in completion_response:
|
| 290 |
+
outputText = completion_response["generations"][0]["text"]
|
| 291 |
+
model_response.choices[0].finish_reason = map_finish_reason(
|
| 292 |
+
completion_response["generations"][0]["finish_reason"]
|
| 293 |
+
)
|
| 294 |
+
elif provider == "anthropic":
|
| 295 |
+
return litellm.AmazonAnthropicClaudeConfig().transform_response(
|
| 296 |
+
model=model,
|
| 297 |
+
raw_response=raw_response,
|
| 298 |
+
model_response=model_response,
|
| 299 |
+
logging_obj=logging_obj,
|
| 300 |
+
request_data=request_data,
|
| 301 |
+
messages=messages,
|
| 302 |
+
optional_params=optional_params,
|
| 303 |
+
litellm_params=litellm_params,
|
| 304 |
+
encoding=encoding,
|
| 305 |
+
api_key=api_key,
|
| 306 |
+
json_mode=json_mode,
|
| 307 |
+
)
|
| 308 |
+
elif provider == "nova":
|
| 309 |
+
return litellm.AmazonInvokeNovaConfig().transform_response(
|
| 310 |
+
model=model,
|
| 311 |
+
raw_response=raw_response,
|
| 312 |
+
model_response=model_response,
|
| 313 |
+
logging_obj=logging_obj,
|
| 314 |
+
request_data=request_data,
|
| 315 |
+
messages=messages,
|
| 316 |
+
optional_params=optional_params,
|
| 317 |
+
litellm_params=litellm_params,
|
| 318 |
+
encoding=encoding,
|
| 319 |
+
)
|
| 320 |
+
elif provider == "twelvelabs":
|
| 321 |
+
return litellm.AmazonTwelveLabsPegasusConfig().transform_response(
|
| 322 |
+
model=model,
|
| 323 |
+
raw_response=raw_response,
|
| 324 |
+
model_response=model_response,
|
| 325 |
+
logging_obj=logging_obj,
|
| 326 |
+
request_data=request_data,
|
| 327 |
+
messages=messages,
|
| 328 |
+
optional_params=optional_params,
|
| 329 |
+
litellm_params=litellm_params,
|
| 330 |
+
encoding=encoding,
|
| 331 |
+
api_key=api_key,
|
| 332 |
+
json_mode=json_mode,
|
| 333 |
+
)
|
| 334 |
+
elif provider == "ai21":
|
| 335 |
+
outputText = (
|
| 336 |
+
completion_response.get("completions")[0].get("data").get("text")
|
| 337 |
+
)
|
| 338 |
+
elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
|
| 339 |
+
outputText = completion_response["generation"]
|
| 340 |
+
elif provider == "mistral":
|
| 341 |
+
outputText = litellm.AmazonMistralConfig.get_outputText(
|
| 342 |
+
completion_response, model_response
|
| 343 |
+
)
|
| 344 |
+
else: # amazon titan
|
| 345 |
+
outputText = completion_response.get("results")[0].get("outputText")
|
| 346 |
+
except Exception as e:
|
| 347 |
+
raise BedrockError(
|
| 348 |
+
message="Error processing={}, Received error={}".format(
|
| 349 |
+
raw_response.text, str(e)
|
| 350 |
+
),
|
| 351 |
+
status_code=422,
|
| 352 |
+
)
|
| 353 |
+
|
| 354 |
+
try:
|
| 355 |
+
if (
|
| 356 |
+
outputText is not None
|
| 357 |
+
and len(outputText) > 0
|
| 358 |
+
and hasattr(model_response.choices[0], "message")
|
| 359 |
+
and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore
|
| 360 |
+
is None
|
| 361 |
+
):
|
| 362 |
+
model_response.choices[0].message.content = outputText # type: ignore
|
| 363 |
+
elif (
|
| 364 |
+
hasattr(model_response.choices[0], "message")
|
| 365 |
+
and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore
|
| 366 |
+
is not None
|
| 367 |
+
):
|
| 368 |
+
pass
|
| 369 |
+
else:
|
| 370 |
+
raise Exception()
|
| 371 |
+
except Exception as e:
|
| 372 |
+
raise BedrockError(
|
| 373 |
+
message="Error parsing received text={}.\nError-{}".format(
|
| 374 |
+
outputText, str(e)
|
| 375 |
+
),
|
| 376 |
+
status_code=raw_response.status_code,
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
## CALCULATING USAGE - bedrock returns usage in the headers
|
| 380 |
+
bedrock_input_tokens = raw_response.headers.get(
|
| 381 |
+
"x-amzn-bedrock-input-token-count", None
|
| 382 |
+
)
|
| 383 |
+
bedrock_output_tokens = raw_response.headers.get(
|
| 384 |
+
"x-amzn-bedrock-output-token-count", None
|
| 385 |
+
)
|
| 386 |
+
|
| 387 |
+
prompt_tokens = int(
|
| 388 |
+
bedrock_input_tokens or litellm.token_counter(messages=messages)
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
completion_tokens = int(
|
| 392 |
+
bedrock_output_tokens
|
| 393 |
+
or litellm.token_counter(
|
| 394 |
+
text=model_response.choices[0].message.content, # type: ignore
|
| 395 |
+
count_response_tokens=True,
|
| 396 |
+
)
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
model_response.created = int(time.time())
|
| 400 |
+
model_response.model = model
|
| 401 |
+
usage = Usage(
|
| 402 |
+
prompt_tokens=prompt_tokens,
|
| 403 |
+
completion_tokens=completion_tokens,
|
| 404 |
+
total_tokens=prompt_tokens + completion_tokens,
|
| 405 |
+
)
|
| 406 |
+
setattr(model_response, "usage", usage)
|
| 407 |
+
|
| 408 |
+
return model_response
|
| 409 |
+
|
| 410 |
+
def validate_environment(
|
| 411 |
+
self,
|
| 412 |
+
headers: dict,
|
| 413 |
+
model: str,
|
| 414 |
+
messages: List[AllMessageValues],
|
| 415 |
+
optional_params: dict,
|
| 416 |
+
litellm_params: dict,
|
| 417 |
+
api_key: Optional[str] = None,
|
| 418 |
+
api_base: Optional[str] = None,
|
| 419 |
+
) -> dict:
|
| 420 |
+
return headers
|
| 421 |
+
|
| 422 |
+
def get_error_class(
|
| 423 |
+
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
| 424 |
+
) -> BaseLLMException:
|
| 425 |
+
return BedrockError(status_code=status_code, message=error_message)
|
| 426 |
+
|
| 427 |
+
@track_llm_api_timing()
|
| 428 |
+
async def get_async_custom_stream_wrapper(
|
| 429 |
+
self,
|
| 430 |
+
model: str,
|
| 431 |
+
custom_llm_provider: str,
|
| 432 |
+
logging_obj: LiteLLMLoggingObj,
|
| 433 |
+
api_base: str,
|
| 434 |
+
headers: dict,
|
| 435 |
+
data: dict,
|
| 436 |
+
messages: list,
|
| 437 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 438 |
+
json_mode: Optional[bool] = None,
|
| 439 |
+
signed_json_body: Optional[bytes] = None,
|
| 440 |
+
) -> CustomStreamWrapper:
|
| 441 |
+
streaming_response = CustomStreamWrapper(
|
| 442 |
+
completion_stream=None,
|
| 443 |
+
make_call=partial(
|
| 444 |
+
make_call,
|
| 445 |
+
client=client,
|
| 446 |
+
api_base=api_base,
|
| 447 |
+
headers=headers,
|
| 448 |
+
data=json.dumps(data),
|
| 449 |
+
model=model,
|
| 450 |
+
messages=messages,
|
| 451 |
+
logging_obj=logging_obj,
|
| 452 |
+
fake_stream=True if "ai21" in api_base else False,
|
| 453 |
+
bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
|
| 454 |
+
json_mode=json_mode,
|
| 455 |
+
),
|
| 456 |
+
model=model,
|
| 457 |
+
custom_llm_provider="bedrock",
|
| 458 |
+
logging_obj=logging_obj,
|
| 459 |
+
)
|
| 460 |
+
return streaming_response
|
| 461 |
+
|
| 462 |
+
@track_llm_api_timing()
|
| 463 |
+
def get_sync_custom_stream_wrapper(
|
| 464 |
+
self,
|
| 465 |
+
model: str,
|
| 466 |
+
custom_llm_provider: str,
|
| 467 |
+
logging_obj: LiteLLMLoggingObj,
|
| 468 |
+
api_base: str,
|
| 469 |
+
headers: dict,
|
| 470 |
+
data: dict,
|
| 471 |
+
messages: list,
|
| 472 |
+
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
| 473 |
+
json_mode: Optional[bool] = None,
|
| 474 |
+
signed_json_body: Optional[bytes] = None,
|
| 475 |
+
) -> CustomStreamWrapper:
|
| 476 |
+
if client is None or isinstance(client, AsyncHTTPHandler):
|
| 477 |
+
client = _get_httpx_client(params={})
|
| 478 |
+
streaming_response = CustomStreamWrapper(
|
| 479 |
+
completion_stream=None,
|
| 480 |
+
make_call=partial(
|
| 481 |
+
make_sync_call,
|
| 482 |
+
client=client,
|
| 483 |
+
api_base=api_base,
|
| 484 |
+
headers=headers,
|
| 485 |
+
data=json.dumps(data),
|
| 486 |
+
signed_json_body=signed_json_body,
|
| 487 |
+
model=model,
|
| 488 |
+
messages=messages,
|
| 489 |
+
logging_obj=logging_obj,
|
| 490 |
+
fake_stream=True if "ai21" in api_base else False,
|
| 491 |
+
bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
|
| 492 |
+
json_mode=json_mode,
|
| 493 |
+
),
|
| 494 |
+
model=model,
|
| 495 |
+
custom_llm_provider="bedrock",
|
| 496 |
+
logging_obj=logging_obj,
|
| 497 |
+
)
|
| 498 |
+
return streaming_response
|
| 499 |
+
|
| 500 |
+
@property
|
| 501 |
+
def has_custom_stream_wrapper(self) -> bool:
|
| 502 |
+
return True
|
| 503 |
+
|
| 504 |
+
@property
|
| 505 |
+
def supports_stream_param_in_request_body(self) -> bool:
|
| 506 |
+
"""
|
| 507 |
+
Bedrock invoke does not allow passing `stream` in the request body.
|
| 508 |
+
"""
|
| 509 |
+
return False
|
| 510 |
+
|
| 511 |
+
@staticmethod
|
| 512 |
+
def get_bedrock_invoke_provider(
|
| 513 |
+
model: str,
|
| 514 |
+
) -> Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL]:
|
| 515 |
+
"""
|
| 516 |
+
Helper function to get the bedrock provider from the model
|
| 517 |
+
|
| 518 |
+
handles 4 scenarios:
|
| 519 |
+
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
| 520 |
+
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
| 521 |
+
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
|
| 522 |
+
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
|
| 523 |
+
"""
|
| 524 |
+
if model.startswith("invoke/"):
|
| 525 |
+
model = model.replace("invoke/", "", 1)
|
| 526 |
+
|
| 527 |
+
# Special case: Check for "nova" in model name first (before "amazon")
|
| 528 |
+
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
|
| 529 |
+
if "nova" in model.lower():
|
| 530 |
+
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 531 |
+
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
|
| 532 |
+
|
| 533 |
+
_split_model = model.split(".")[0]
|
| 534 |
+
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 535 |
+
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
|
| 536 |
+
|
| 537 |
+
# If not a known provider, check for pattern with two slashes
|
| 538 |
+
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
|
| 539 |
+
if provider is not None:
|
| 540 |
+
return provider
|
| 541 |
+
|
| 542 |
+
for provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 543 |
+
if provider in model:
|
| 544 |
+
return provider
|
| 545 |
+
return None
|
| 546 |
+
|
| 547 |
+
@staticmethod
|
| 548 |
+
def _get_provider_from_model_path(
|
| 549 |
+
model_path: str,
|
| 550 |
+
) -> Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL]:
|
| 551 |
+
"""
|
| 552 |
+
Helper function to get the provider from a model path with format: provider/model-name
|
| 553 |
+
|
| 554 |
+
Args:
|
| 555 |
+
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
|
| 556 |
+
|
| 557 |
+
Returns:
|
| 558 |
+
Optional[str]: The provider name, or None if no valid provider found
|
| 559 |
+
"""
|
| 560 |
+
parts = model_path.split("/")
|
| 561 |
+
if len(parts) >= 1:
|
| 562 |
+
provider = parts[0]
|
| 563 |
+
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
| 564 |
+
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
|
| 565 |
+
return None
|
| 566 |
+
|
| 567 |
+
def convert_messages_to_prompt(
|
| 568 |
+
self, model, messages, provider, custom_prompt_dict
|
| 569 |
+
) -> Tuple[str, Optional[list]]:
|
| 570 |
+
# handle anthropic prompts and amazon titan prompts
|
| 571 |
+
prompt = ""
|
| 572 |
+
chat_history: Optional[list] = None
|
| 573 |
+
## CUSTOM PROMPT
|
| 574 |
+
if model in custom_prompt_dict:
|
| 575 |
+
# check if the model has a registered custom prompt
|
| 576 |
+
model_prompt_details = custom_prompt_dict[model]
|
| 577 |
+
prompt = custom_prompt(
|
| 578 |
+
role_dict=model_prompt_details["roles"],
|
| 579 |
+
initial_prompt_value=model_prompt_details.get(
|
| 580 |
+
"initial_prompt_value", ""
|
| 581 |
+
),
|
| 582 |
+
final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
|
| 583 |
+
messages=messages,
|
| 584 |
+
)
|
| 585 |
+
return prompt, None
|
| 586 |
+
## ELSE
|
| 587 |
+
if provider == "anthropic" or provider == "amazon":
|
| 588 |
+
prompt = prompt_factory(
|
| 589 |
+
model=model, messages=messages, custom_llm_provider="bedrock"
|
| 590 |
+
)
|
| 591 |
+
elif provider == "mistral":
|
| 592 |
+
prompt = prompt_factory(
|
| 593 |
+
model=model, messages=messages, custom_llm_provider="bedrock"
|
| 594 |
+
)
|
| 595 |
+
elif provider == "meta" or provider == "llama":
|
| 596 |
+
prompt = prompt_factory(
|
| 597 |
+
model=model, messages=messages, custom_llm_provider="bedrock"
|
| 598 |
+
)
|
| 599 |
+
elif provider == "cohere":
|
| 600 |
+
prompt, chat_history = cohere_message_pt(messages=messages)
|
| 601 |
+
elif provider == "deepseek_r1":
|
| 602 |
+
prompt = deepseek_r1_pt(messages=messages)
|
| 603 |
+
else:
|
| 604 |
+
prompt = ""
|
| 605 |
+
for message in messages:
|
| 606 |
+
if "role" in message:
|
| 607 |
+
if message["role"] == "user":
|
| 608 |
+
prompt += f"{message['content']}"
|
| 609 |
+
else:
|
| 610 |
+
prompt += f"{message['content']}"
|
| 611 |
+
else:
|
| 612 |
+
prompt += f"{message['content']}"
|
| 613 |
+
return prompt, chat_history # type: ignore
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/common_utils.py
ADDED
|
@@ -0,0 +1,1140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
"""
|
| 4 |
+
Common utilities used across bedrock chat/embedding/image generation
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Union
|
| 10 |
+
|
| 11 |
+
if TYPE_CHECKING:
|
| 12 |
+
from litellm.types.llms.bedrock import BedrockCreateBatchRequest
|
| 13 |
+
|
| 14 |
+
import httpx
|
| 15 |
+
|
| 16 |
+
import litellm
|
| 17 |
+
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
| 18 |
+
BaseAnthropicMessagesConfig,
|
| 19 |
+
)
|
| 20 |
+
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
| 21 |
+
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
| 22 |
+
from litellm.secret_managers.main import get_secret
|
| 23 |
+
|
| 24 |
+
if TYPE_CHECKING:
|
| 25 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class BedrockError(BaseLLMException):
|
| 29 |
+
pass
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
# Lazy import cache to avoid circular imports and performance impact
|
| 33 |
+
_get_model_info = None
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def get_cached_model_info():
|
| 37 |
+
"""
|
| 38 |
+
Lazy import and cache get_model_info to avoid circular imports.
|
| 39 |
+
|
| 40 |
+
This function is used by bedrock transformation classes that need get_model_info
|
| 41 |
+
but cannot import it at module level due to circular import issues.
|
| 42 |
+
The function is cached after first use to avoid performance impact.
|
| 43 |
+
"""
|
| 44 |
+
global _get_model_info
|
| 45 |
+
if _get_model_info is None:
|
| 46 |
+
from litellm import get_model_info
|
| 47 |
+
|
| 48 |
+
_get_model_info = get_model_info
|
| 49 |
+
return _get_model_info
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def remove_custom_field_from_tools(request_body: dict) -> None:
|
| 53 |
+
"""
|
| 54 |
+
Remove ``custom`` field from each tool in the request body.
|
| 55 |
+
|
| 56 |
+
Claude Code (v2.1.69+) sends ``custom: {defer_loading: true}`` on tool
|
| 57 |
+
definitions, which Anthropic's API accepts but Bedrock rejects with
|
| 58 |
+
``"Extra inputs are not permitted"``.
|
| 59 |
+
|
| 60 |
+
Args:
|
| 61 |
+
request_body: The request dictionary to modify in-place.
|
| 62 |
+
|
| 63 |
+
Ref: https://github.com/BerriAI/litellm/issues/22847
|
| 64 |
+
"""
|
| 65 |
+
tools = request_body.get("tools")
|
| 66 |
+
if not tools or not isinstance(tools, list):
|
| 67 |
+
return
|
| 68 |
+
for tool in tools:
|
| 69 |
+
if isinstance(tool, dict):
|
| 70 |
+
tool.pop("custom", None)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class AmazonBedrockGlobalConfig:
|
| 74 |
+
def __init__(self):
|
| 75 |
+
pass
|
| 76 |
+
|
| 77 |
+
def get_mapped_special_auth_params(self) -> dict:
|
| 78 |
+
"""
|
| 79 |
+
Mapping of common auth params across bedrock/vertex/azure/watsonx
|
| 80 |
+
"""
|
| 81 |
+
return {"region_name": "aws_region_name"}
|
| 82 |
+
|
| 83 |
+
def map_special_auth_params(self, non_default_params: dict, optional_params: dict):
|
| 84 |
+
mapped_params = self.get_mapped_special_auth_params()
|
| 85 |
+
for param, value in non_default_params.items():
|
| 86 |
+
if param in mapped_params:
|
| 87 |
+
optional_params[mapped_params[param]] = value
|
| 88 |
+
return optional_params
|
| 89 |
+
|
| 90 |
+
def get_all_regions(self) -> List[str]:
|
| 91 |
+
return (
|
| 92 |
+
self.get_us_regions()
|
| 93 |
+
+ self.get_eu_regions()
|
| 94 |
+
+ self.get_ap_regions()
|
| 95 |
+
+ self.get_ca_regions()
|
| 96 |
+
+ self.get_sa_regions()
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
def get_ap_regions(self) -> List[str]:
|
| 100 |
+
"""
|
| 101 |
+
Source: https://www.aws-services.info/bedrock.html
|
| 102 |
+
"""
|
| 103 |
+
return [
|
| 104 |
+
"ap-northeast-1", # Asia Pacific (Tokyo)
|
| 105 |
+
"ap-northeast-2", # Asia Pacific (Seoul)
|
| 106 |
+
"ap-northeast-3", # Asia Pacific (Osaka)
|
| 107 |
+
"ap-south-1", # Asia Pacific (Mumbai)
|
| 108 |
+
"ap-south-2", # Asia Pacific (Hyderabad)
|
| 109 |
+
"ap-southeast-1", # Asia Pacific (Singapore)
|
| 110 |
+
"ap-southeast-2", # Asia Pacific (Sydney)
|
| 111 |
+
]
|
| 112 |
+
|
| 113 |
+
def get_sa_regions(self) -> List[str]:
|
| 114 |
+
return ["sa-east-1"]
|
| 115 |
+
|
| 116 |
+
def get_eu_regions(self) -> List[str]:
|
| 117 |
+
"""
|
| 118 |
+
Source: https://www.aws-services.info/bedrock.html
|
| 119 |
+
"""
|
| 120 |
+
return [
|
| 121 |
+
"eu-west-1", # Europe (Ireland)
|
| 122 |
+
"eu-west-2", # Europe (London)
|
| 123 |
+
"eu-west-3", # Europe (Paris)
|
| 124 |
+
"eu-central-1", # Europe (Frankfurt)
|
| 125 |
+
"eu-central-2", # Europe (Zurich)
|
| 126 |
+
"eu-south-1", # Europe (Milan)
|
| 127 |
+
"eu-south-2", # Europe (Spain)
|
| 128 |
+
"eu-north-1", # Europe (Stockholm)
|
| 129 |
+
]
|
| 130 |
+
|
| 131 |
+
def get_ca_regions(self) -> List[str]:
|
| 132 |
+
return ["ca-central-1"]
|
| 133 |
+
|
| 134 |
+
def get_us_regions(self) -> List[str]:
|
| 135 |
+
"""
|
| 136 |
+
Source: https://www.aws-services.info/bedrock.html
|
| 137 |
+
"""
|
| 138 |
+
return [
|
| 139 |
+
"us-east-1", # US East (N. Virginia)
|
| 140 |
+
"us-east-2", # US East (Ohio)
|
| 141 |
+
"us-west-1", # US West (N. California)
|
| 142 |
+
"us-west-2", # US West (Oregon)
|
| 143 |
+
"us-gov-east-1", # AWS GovCloud (US-East)
|
| 144 |
+
"us-gov-west-1", # AWS GovCloud (US-West)
|
| 145 |
+
]
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def add_custom_header(headers):
|
| 149 |
+
"""Closure to capture the headers and add them."""
|
| 150 |
+
|
| 151 |
+
def callback(request, **kwargs):
|
| 152 |
+
"""Actual callback function that Boto3 will call."""
|
| 153 |
+
for header_name, header_value in headers.items():
|
| 154 |
+
request.headers.add_header(header_name, header_value)
|
| 155 |
+
|
| 156 |
+
return callback
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _get_bedrock_client_ssl_verify() -> Union[bool, str]:
|
| 160 |
+
"""
|
| 161 |
+
Get SSL verification setting for Bedrock client.
|
| 162 |
+
|
| 163 |
+
Returns the SSL verification setting which can be:
|
| 164 |
+
- True: Use default SSL verification
|
| 165 |
+
- False: Disable SSL verification
|
| 166 |
+
- str: Path to a custom CA bundle file
|
| 167 |
+
"""
|
| 168 |
+
from litellm.llms.custom_httpx.http_handler import get_ssl_verify
|
| 169 |
+
|
| 170 |
+
return get_ssl_verify()
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def init_bedrock_client(
|
| 174 |
+
region_name=None,
|
| 175 |
+
aws_access_key_id: Optional[str] = None,
|
| 176 |
+
aws_secret_access_key: Optional[str] = None,
|
| 177 |
+
aws_region_name: Optional[str] = None,
|
| 178 |
+
aws_bedrock_runtime_endpoint: Optional[str] = None,
|
| 179 |
+
aws_session_name: Optional[str] = None,
|
| 180 |
+
aws_profile_name: Optional[str] = None,
|
| 181 |
+
aws_role_name: Optional[str] = None,
|
| 182 |
+
aws_web_identity_token: Optional[str] = None,
|
| 183 |
+
extra_headers: Optional[dict] = None,
|
| 184 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 185 |
+
):
|
| 186 |
+
# check for custom AWS_REGION_NAME and use it if not passed to init_bedrock_client
|
| 187 |
+
litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
|
| 188 |
+
standard_aws_region_name = get_secret("AWS_REGION", None)
|
| 189 |
+
## CHECK IS 'os.environ/' passed in
|
| 190 |
+
# Define the list of parameters to check
|
| 191 |
+
params_to_check = [
|
| 192 |
+
aws_access_key_id,
|
| 193 |
+
aws_secret_access_key,
|
| 194 |
+
aws_region_name,
|
| 195 |
+
aws_bedrock_runtime_endpoint,
|
| 196 |
+
aws_session_name,
|
| 197 |
+
aws_profile_name,
|
| 198 |
+
aws_role_name,
|
| 199 |
+
aws_web_identity_token,
|
| 200 |
+
]
|
| 201 |
+
|
| 202 |
+
# Iterate over parameters and update if needed
|
| 203 |
+
for i, param in enumerate(params_to_check):
|
| 204 |
+
if param and param.startswith("os.environ/"):
|
| 205 |
+
params_to_check[i] = get_secret(param) # type: ignore
|
| 206 |
+
# Assign updated values back to parameters
|
| 207 |
+
(
|
| 208 |
+
aws_access_key_id,
|
| 209 |
+
aws_secret_access_key,
|
| 210 |
+
aws_region_name,
|
| 211 |
+
aws_bedrock_runtime_endpoint,
|
| 212 |
+
aws_session_name,
|
| 213 |
+
aws_profile_name,
|
| 214 |
+
aws_role_name,
|
| 215 |
+
aws_web_identity_token,
|
| 216 |
+
) = params_to_check
|
| 217 |
+
|
| 218 |
+
ssl_verify = _get_bedrock_client_ssl_verify()
|
| 219 |
+
|
| 220 |
+
### SET REGION NAME
|
| 221 |
+
if region_name:
|
| 222 |
+
pass
|
| 223 |
+
elif aws_region_name:
|
| 224 |
+
region_name = aws_region_name
|
| 225 |
+
elif litellm_aws_region_name:
|
| 226 |
+
region_name = litellm_aws_region_name
|
| 227 |
+
elif standard_aws_region_name:
|
| 228 |
+
region_name = standard_aws_region_name
|
| 229 |
+
else:
|
| 230 |
+
raise BedrockError(
|
| 231 |
+
message="AWS region not set: set AWS_REGION_NAME or AWS_REGION env variable or in .env file",
|
| 232 |
+
status_code=401,
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
# check for custom AWS_BEDROCK_RUNTIME_ENDPOINT and use it if not passed to init_bedrock_client
|
| 236 |
+
env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT")
|
| 237 |
+
if aws_bedrock_runtime_endpoint:
|
| 238 |
+
endpoint_url = aws_bedrock_runtime_endpoint
|
| 239 |
+
elif env_aws_bedrock_runtime_endpoint:
|
| 240 |
+
endpoint_url = env_aws_bedrock_runtime_endpoint
|
| 241 |
+
else:
|
| 242 |
+
endpoint_url = f"https://bedrock-runtime.{region_name}.amazonaws.com"
|
| 243 |
+
|
| 244 |
+
import boto3
|
| 245 |
+
|
| 246 |
+
if isinstance(timeout, float):
|
| 247 |
+
config = boto3.session.Config(connect_timeout=timeout, read_timeout=timeout) # type: ignore
|
| 248 |
+
elif isinstance(timeout, httpx.Timeout):
|
| 249 |
+
config = boto3.session.Config( # type: ignore
|
| 250 |
+
connect_timeout=timeout.connect, read_timeout=timeout.read
|
| 251 |
+
)
|
| 252 |
+
else:
|
| 253 |
+
config = boto3.session.Config() # type: ignore
|
| 254 |
+
|
| 255 |
+
### CHECK STS ###
|
| 256 |
+
if (
|
| 257 |
+
aws_web_identity_token is not None
|
| 258 |
+
and aws_role_name is not None
|
| 259 |
+
and aws_session_name is not None
|
| 260 |
+
):
|
| 261 |
+
oidc_token = get_secret(aws_web_identity_token)
|
| 262 |
+
|
| 263 |
+
if oidc_token is None:
|
| 264 |
+
raise BedrockError(
|
| 265 |
+
message="OIDC token could not be retrieved from secret manager.",
|
| 266 |
+
status_code=401,
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
sts_client = boto3.client("sts", verify=ssl_verify)
|
| 270 |
+
|
| 271 |
+
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
|
| 272 |
+
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
|
| 273 |
+
sts_response = sts_client.assume_role_with_web_identity(
|
| 274 |
+
RoleArn=aws_role_name,
|
| 275 |
+
RoleSessionName=aws_session_name,
|
| 276 |
+
WebIdentityToken=oidc_token,
|
| 277 |
+
DurationSeconds=3600,
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
client = boto3.client(
|
| 281 |
+
service_name="bedrock-runtime",
|
| 282 |
+
aws_access_key_id=sts_response["Credentials"]["AccessKeyId"],
|
| 283 |
+
aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"],
|
| 284 |
+
aws_session_token=sts_response["Credentials"]["SessionToken"],
|
| 285 |
+
region_name=region_name,
|
| 286 |
+
endpoint_url=endpoint_url,
|
| 287 |
+
config=config,
|
| 288 |
+
verify=ssl_verify,
|
| 289 |
+
)
|
| 290 |
+
elif aws_role_name is not None and aws_session_name is not None:
|
| 291 |
+
# use sts if role name passed in
|
| 292 |
+
sts_client = boto3.client(
|
| 293 |
+
"sts",
|
| 294 |
+
aws_access_key_id=aws_access_key_id,
|
| 295 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 296 |
+
verify=ssl_verify,
|
| 297 |
+
)
|
| 298 |
+
|
| 299 |
+
sts_response = sts_client.assume_role(
|
| 300 |
+
RoleArn=aws_role_name, RoleSessionName=aws_session_name
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
client = boto3.client(
|
| 304 |
+
service_name="bedrock-runtime",
|
| 305 |
+
aws_access_key_id=sts_response["Credentials"]["AccessKeyId"],
|
| 306 |
+
aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"],
|
| 307 |
+
aws_session_token=sts_response["Credentials"]["SessionToken"],
|
| 308 |
+
region_name=region_name,
|
| 309 |
+
endpoint_url=endpoint_url,
|
| 310 |
+
config=config,
|
| 311 |
+
verify=ssl_verify,
|
| 312 |
+
)
|
| 313 |
+
elif aws_access_key_id is not None:
|
| 314 |
+
# uses auth params passed to completion
|
| 315 |
+
# aws_access_key_id is not None, assume user is trying to auth using litellm.completion
|
| 316 |
+
|
| 317 |
+
client = boto3.client(
|
| 318 |
+
service_name="bedrock-runtime",
|
| 319 |
+
aws_access_key_id=aws_access_key_id,
|
| 320 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 321 |
+
region_name=region_name,
|
| 322 |
+
endpoint_url=endpoint_url,
|
| 323 |
+
config=config,
|
| 324 |
+
verify=ssl_verify,
|
| 325 |
+
)
|
| 326 |
+
elif aws_profile_name is not None:
|
| 327 |
+
# uses auth values from AWS profile usually stored in ~/.aws/credentials
|
| 328 |
+
|
| 329 |
+
client = boto3.Session(profile_name=aws_profile_name).client(
|
| 330 |
+
service_name="bedrock-runtime",
|
| 331 |
+
region_name=region_name,
|
| 332 |
+
endpoint_url=endpoint_url,
|
| 333 |
+
config=config,
|
| 334 |
+
verify=ssl_verify,
|
| 335 |
+
)
|
| 336 |
+
else:
|
| 337 |
+
# aws_access_key_id is None, assume user is trying to auth using env variables
|
| 338 |
+
# boto3 automatically reads env variables
|
| 339 |
+
|
| 340 |
+
client = boto3.client(
|
| 341 |
+
service_name="bedrock-runtime",
|
| 342 |
+
region_name=region_name,
|
| 343 |
+
endpoint_url=endpoint_url,
|
| 344 |
+
config=config,
|
| 345 |
+
verify=ssl_verify,
|
| 346 |
+
)
|
| 347 |
+
if extra_headers:
|
| 348 |
+
client.meta.events.register(
|
| 349 |
+
"before-sign.bedrock-runtime.*", add_custom_header(extra_headers)
|
| 350 |
+
)
|
| 351 |
+
|
| 352 |
+
return client
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
class ModelResponseIterator:
|
| 356 |
+
def __init__(self, model_response):
|
| 357 |
+
self.model_response = model_response
|
| 358 |
+
self.is_done = False
|
| 359 |
+
|
| 360 |
+
# Sync iterator
|
| 361 |
+
def __iter__(self):
|
| 362 |
+
return self
|
| 363 |
+
|
| 364 |
+
def __next__(self):
|
| 365 |
+
if self.is_done:
|
| 366 |
+
raise StopIteration
|
| 367 |
+
self.is_done = True
|
| 368 |
+
return self.model_response
|
| 369 |
+
|
| 370 |
+
# Async iterator
|
| 371 |
+
def __aiter__(self):
|
| 372 |
+
return self
|
| 373 |
+
|
| 374 |
+
async def __anext__(self):
|
| 375 |
+
if self.is_done:
|
| 376 |
+
raise StopAsyncIteration
|
| 377 |
+
self.is_done = True
|
| 378 |
+
return self.model_response
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def get_bedrock_tool_name(response_tool_name: str) -> str:
|
| 382 |
+
"""
|
| 383 |
+
If litellm formatted the input tool name, we need to convert it back to the original name.
|
| 384 |
+
|
| 385 |
+
Args:
|
| 386 |
+
response_tool_name (str): The name of the tool as received from the response.
|
| 387 |
+
|
| 388 |
+
Returns:
|
| 389 |
+
str: The original name of the tool.
|
| 390 |
+
"""
|
| 391 |
+
|
| 392 |
+
if response_tool_name in litellm.bedrock_tool_name_mappings.cache_dict:
|
| 393 |
+
response_tool_name = litellm.bedrock_tool_name_mappings.cache_dict[
|
| 394 |
+
response_tool_name
|
| 395 |
+
]
|
| 396 |
+
return response_tool_name
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
# Cache the global regions list at module level
|
| 400 |
+
_BEDROCK_GLOBAL_REGIONS: Optional[List[str]] = None
|
| 401 |
+
|
| 402 |
+
|
| 403 |
+
def _get_all_bedrock_regions() -> List[str]:
|
| 404 |
+
"""Get all Bedrock regions, cached at module level."""
|
| 405 |
+
global _BEDROCK_GLOBAL_REGIONS
|
| 406 |
+
if _BEDROCK_GLOBAL_REGIONS is None:
|
| 407 |
+
_BEDROCK_GLOBAL_REGIONS = AmazonBedrockGlobalConfig().get_all_regions()
|
| 408 |
+
return _BEDROCK_GLOBAL_REGIONS
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
def get_bedrock_cross_region_inference_regions() -> List[str]:
|
| 412 |
+
"""Abbreviations of regions AWS Bedrock supports for cross region inference."""
|
| 413 |
+
return ["global", "us", "eu", "apac", "jp", "au", "us-gov"]
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
def extract_model_name_from_bedrock_arn(model: str) -> str:
|
| 417 |
+
"""
|
| 418 |
+
Extract the model name from an AWS Bedrock ARN.
|
| 419 |
+
Returns the string after the last '/' if 'arn' is in the input string.
|
| 420 |
+
"""
|
| 421 |
+
if "arn" in model.lower():
|
| 422 |
+
return model.split("/")[-1]
|
| 423 |
+
return model
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
def strip_bedrock_routing_prefix(model: str) -> str:
|
| 427 |
+
"""Strip LiteLLM routing prefixes from model name."""
|
| 428 |
+
for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "nova-2/", "nova/"]:
|
| 429 |
+
if model.startswith(prefix):
|
| 430 |
+
model = model.split("/", 1)[1]
|
| 431 |
+
return model
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
def strip_bedrock_throughput_suffix(model: str) -> str:
|
| 435 |
+
"""Strip throughput tier suffixes from Bedrock model names."""
|
| 436 |
+
import re
|
| 437 |
+
|
| 438 |
+
# Pattern matches model:version:throughput where throughput is like 51k, 18k, etc.
|
| 439 |
+
# Keep the model:version part, strip the :throughput suffix
|
| 440 |
+
return re.sub(r"(:\d+):\d+k$", r"\1", model)
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
def get_bedrock_base_model(model: str) -> str:
|
| 444 |
+
"""
|
| 445 |
+
Get the base model from the given model name.
|
| 446 |
+
|
| 447 |
+
Handle model names like:
|
| 448 |
+
- "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1"
|
| 449 |
+
- "bedrock/converse/model" -> "model"
|
| 450 |
+
- "anthropic.claude-3-5-sonnet-20241022-v2:0:51k" -> "anthropic.claude-3-5-sonnet-20241022-v2:0"
|
| 451 |
+
- "bedrock/nova-2/arn:aws:..." -> "amazon.nova-2-custom"
|
| 452 |
+
- "bedrock/nova/arn:aws:..." -> "amazon.nova-custom"
|
| 453 |
+
"""
|
| 454 |
+
# Detect nova spec prefixes before stripping them
|
| 455 |
+
stripped = model
|
| 456 |
+
for rp in ["bedrock/converse/", "bedrock/", "converse/"]:
|
| 457 |
+
if stripped.startswith(rp):
|
| 458 |
+
stripped = stripped[len(rp) :]
|
| 459 |
+
break
|
| 460 |
+
if stripped.startswith("nova-2/"):
|
| 461 |
+
return "amazon.nova-2-custom"
|
| 462 |
+
elif stripped.startswith("nova/"):
|
| 463 |
+
return "amazon.nova-custom"
|
| 464 |
+
|
| 465 |
+
model = strip_bedrock_routing_prefix(model)
|
| 466 |
+
model = extract_model_name_from_bedrock_arn(model)
|
| 467 |
+
model = strip_bedrock_throughput_suffix(model)
|
| 468 |
+
|
| 469 |
+
potential_region = model.split(".", 1)[0]
|
| 470 |
+
alt_potential_region = model.split("/", 1)[0]
|
| 471 |
+
|
| 472 |
+
if potential_region in get_bedrock_cross_region_inference_regions():
|
| 473 |
+
return model.split(".", 1)[1]
|
| 474 |
+
elif (
|
| 475 |
+
alt_potential_region in _get_all_bedrock_regions()
|
| 476 |
+
and len(model.split("/", 1)) > 1
|
| 477 |
+
):
|
| 478 |
+
return model.split("/", 1)[1]
|
| 479 |
+
|
| 480 |
+
return model
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
def is_claude_4_5_on_bedrock(model: str) -> bool:
|
| 484 |
+
"""
|
| 485 |
+
Check if the model is a Claude 4.5 model on Bedrock.
|
| 486 |
+
Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
|
| 487 |
+
"""
|
| 488 |
+
model_lower = model.lower()
|
| 489 |
+
claude_4_5_patterns = [
|
| 490 |
+
"sonnet-4.5",
|
| 491 |
+
"sonnet_4.5",
|
| 492 |
+
"sonnet-4-5",
|
| 493 |
+
"sonnet_4_5",
|
| 494 |
+
"haiku-4.5",
|
| 495 |
+
"haiku_4.5",
|
| 496 |
+
"haiku-4-5",
|
| 497 |
+
"haiku_4_5",
|
| 498 |
+
"opus-4.5",
|
| 499 |
+
"opus_4.5",
|
| 500 |
+
"opus-4-5",
|
| 501 |
+
"opus_4_5",
|
| 502 |
+
"sonnet-4.6",
|
| 503 |
+
"sonnet_4.6",
|
| 504 |
+
"sonnet-4-6",
|
| 505 |
+
"sonnet_4_6",
|
| 506 |
+
"opus-4.6",
|
| 507 |
+
"opus_4.6",
|
| 508 |
+
"opus-4-6",
|
| 509 |
+
"opus_4_6",
|
| 510 |
+
]
|
| 511 |
+
return any(pattern in model_lower for pattern in claude_4_5_patterns)
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
# Import after standalone functions to avoid circular imports
|
| 515 |
+
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
class BedrockModelInfo(BaseLLMModelInfo):
|
| 519 |
+
global_config = AmazonBedrockGlobalConfig()
|
| 520 |
+
all_global_regions = global_config.get_all_regions()
|
| 521 |
+
|
| 522 |
+
@staticmethod
|
| 523 |
+
def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
|
| 524 |
+
"""
|
| 525 |
+
Get the API base for the given model.
|
| 526 |
+
"""
|
| 527 |
+
return api_base
|
| 528 |
+
|
| 529 |
+
@staticmethod
|
| 530 |
+
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
|
| 531 |
+
"""
|
| 532 |
+
Get the API key for the given model.
|
| 533 |
+
"""
|
| 534 |
+
return api_key
|
| 535 |
+
|
| 536 |
+
def validate_environment(
|
| 537 |
+
self,
|
| 538 |
+
headers: dict,
|
| 539 |
+
model: str,
|
| 540 |
+
messages: List["AllMessageValues"],
|
| 541 |
+
optional_params: dict,
|
| 542 |
+
litellm_params: dict,
|
| 543 |
+
api_key: Optional[str] = None,
|
| 544 |
+
api_base: Optional[str] = None,
|
| 545 |
+
) -> dict:
|
| 546 |
+
return headers
|
| 547 |
+
|
| 548 |
+
def get_models(
|
| 549 |
+
self, api_key: Optional[str] = None, api_base: Optional[str] = None
|
| 550 |
+
) -> List[str]:
|
| 551 |
+
return []
|
| 552 |
+
|
| 553 |
+
# def get_provider_info(self, model: str) -> Optional[ProviderSpecificModelInfo]:
|
| 554 |
+
# """
|
| 555 |
+
# Handles Bedrock throughput suffixes like ":28k", ":51k".
|
| 556 |
+
# """
|
| 557 |
+
# import re
|
| 558 |
+
|
| 559 |
+
# overrides: ProviderSpecificModelInfo = {}
|
| 560 |
+
|
| 561 |
+
# # Parse context window suffix (e.g., :28k, :51k)
|
| 562 |
+
# match = re.search(r":(\d+)k$", model)
|
| 563 |
+
# if match:
|
| 564 |
+
# throughput_value = int(match.group(1)) * 1000
|
| 565 |
+
# overrides["max_input_tokens"] = throughput_value
|
| 566 |
+
|
| 567 |
+
# return overrides if overrides else None
|
| 568 |
+
|
| 569 |
+
def get_token_counter(self) -> Optional[BaseTokenCounter]:
|
| 570 |
+
"""
|
| 571 |
+
Factory method to create a Bedrock token counter.
|
| 572 |
+
|
| 573 |
+
Returns:
|
| 574 |
+
BedrockTokenCounter instance for this provider.
|
| 575 |
+
"""
|
| 576 |
+
return BedrockTokenCounter()
|
| 577 |
+
|
| 578 |
+
@staticmethod
|
| 579 |
+
def extract_model_name_from_arn(model: str) -> str:
|
| 580 |
+
"""Wrapper for standalone function. See extract_model_name_from_bedrock_arn()."""
|
| 581 |
+
return extract_model_name_from_bedrock_arn(model)
|
| 582 |
+
|
| 583 |
+
@staticmethod
|
| 584 |
+
def get_non_litellm_routing_model_name(model: str) -> str:
|
| 585 |
+
"""Wrapper for standalone function. See strip_bedrock_routing_prefix()."""
|
| 586 |
+
return strip_bedrock_routing_prefix(model)
|
| 587 |
+
|
| 588 |
+
@staticmethod
|
| 589 |
+
def get_base_model(model: str) -> str:
|
| 590 |
+
"""Wrapper for standalone function. See get_bedrock_base_model()."""
|
| 591 |
+
return get_bedrock_base_model(model)
|
| 592 |
+
|
| 593 |
+
@staticmethod
|
| 594 |
+
def _supported_cross_region_inference_region() -> List[str]:
|
| 595 |
+
"""Wrapper for standalone function. See get_bedrock_cross_region_inference_regions()."""
|
| 596 |
+
return get_bedrock_cross_region_inference_regions()
|
| 597 |
+
|
| 598 |
+
@staticmethod
|
| 599 |
+
def get_bedrock_route(
|
| 600 |
+
model: str,
|
| 601 |
+
) -> Literal[
|
| 602 |
+
"converse",
|
| 603 |
+
"invoke",
|
| 604 |
+
"converse_like",
|
| 605 |
+
"agent",
|
| 606 |
+
"agentcore",
|
| 607 |
+
"async_invoke",
|
| 608 |
+
"openai",
|
| 609 |
+
]:
|
| 610 |
+
"""
|
| 611 |
+
Get the bedrock route for the given model.
|
| 612 |
+
"""
|
| 613 |
+
route_mappings: Dict[
|
| 614 |
+
str,
|
| 615 |
+
Literal[
|
| 616 |
+
"invoke",
|
| 617 |
+
"converse_like",
|
| 618 |
+
"converse",
|
| 619 |
+
"agent",
|
| 620 |
+
"agentcore",
|
| 621 |
+
"async_invoke",
|
| 622 |
+
"openai",
|
| 623 |
+
],
|
| 624 |
+
] = {
|
| 625 |
+
"invoke/": "invoke",
|
| 626 |
+
"converse_like/": "converse_like",
|
| 627 |
+
"converse/": "converse",
|
| 628 |
+
"agent/": "agent",
|
| 629 |
+
"agentcore/": "agentcore",
|
| 630 |
+
"async_invoke/": "async_invoke",
|
| 631 |
+
"openai/": "openai",
|
| 632 |
+
}
|
| 633 |
+
|
| 634 |
+
# Check explicit routes first
|
| 635 |
+
for prefix, route_type in route_mappings.items():
|
| 636 |
+
if prefix in model:
|
| 637 |
+
return route_type
|
| 638 |
+
|
| 639 |
+
# Check for nova spec prefixes (nova/ and nova-2/)
|
| 640 |
+
_model_after_bedrock = model.replace("bedrock/", "", 1)
|
| 641 |
+
if _model_after_bedrock.startswith(
|
| 642 |
+
"nova-2/"
|
| 643 |
+
) or _model_after_bedrock.startswith("nova/"):
|
| 644 |
+
return "converse"
|
| 645 |
+
|
| 646 |
+
base_model = BedrockModelInfo.get_base_model(model)
|
| 647 |
+
alt_model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model)
|
| 648 |
+
if (
|
| 649 |
+
base_model in litellm.bedrock_converse_models
|
| 650 |
+
or alt_model in litellm.bedrock_converse_models
|
| 651 |
+
):
|
| 652 |
+
return "converse"
|
| 653 |
+
return "invoke"
|
| 654 |
+
|
| 655 |
+
@staticmethod
|
| 656 |
+
def _explicit_converse_route(model: str) -> bool:
|
| 657 |
+
"""
|
| 658 |
+
Check if the model is an explicit converse route.
|
| 659 |
+
"""
|
| 660 |
+
return "converse/" in model
|
| 661 |
+
|
| 662 |
+
@staticmethod
|
| 663 |
+
def _explicit_invoke_route(model: str) -> bool:
|
| 664 |
+
"""
|
| 665 |
+
Check if the model is an explicit invoke route.
|
| 666 |
+
"""
|
| 667 |
+
return "invoke/" in model
|
| 668 |
+
|
| 669 |
+
@staticmethod
|
| 670 |
+
def _explicit_agent_route(model: str) -> bool:
|
| 671 |
+
"""
|
| 672 |
+
Check if the model is an explicit agent route.
|
| 673 |
+
"""
|
| 674 |
+
return "agent/" in model
|
| 675 |
+
|
| 676 |
+
@staticmethod
|
| 677 |
+
def _explicit_agentcore_route(model: str) -> bool:
|
| 678 |
+
"""
|
| 679 |
+
Check if the model is an explicit agentcore route.
|
| 680 |
+
"""
|
| 681 |
+
return "agentcore/" in model
|
| 682 |
+
|
| 683 |
+
@staticmethod
|
| 684 |
+
def _explicit_converse_like_route(model: str) -> bool:
|
| 685 |
+
"""
|
| 686 |
+
Check if the model is an explicit converse like route.
|
| 687 |
+
"""
|
| 688 |
+
return "converse_like/" in model
|
| 689 |
+
|
| 690 |
+
@staticmethod
|
| 691 |
+
def _explicit_async_invoke_route(model: str) -> bool:
|
| 692 |
+
"""
|
| 693 |
+
Check if the model is an explicit async invoke route.
|
| 694 |
+
"""
|
| 695 |
+
return "async_invoke/" in model
|
| 696 |
+
|
| 697 |
+
@staticmethod
|
| 698 |
+
def _explicit_openai_route(model: str) -> bool:
|
| 699 |
+
"""
|
| 700 |
+
Check if the model is an explicit openai route.
|
| 701 |
+
Used for Bedrock imported models that use OpenAI Chat Completions format.
|
| 702 |
+
"""
|
| 703 |
+
return "openai/" in model
|
| 704 |
+
|
| 705 |
+
@staticmethod
|
| 706 |
+
def get_bedrock_provider_config_for_messages_api(
|
| 707 |
+
model: str,
|
| 708 |
+
) -> Optional[BaseAnthropicMessagesConfig]:
|
| 709 |
+
"""
|
| 710 |
+
Get the bedrock provider config for the given model.
|
| 711 |
+
|
| 712 |
+
Only route to AmazonAnthropicClaude3MessagesConfig() for BaseMessagesConfig
|
| 713 |
+
|
| 714 |
+
All other routes should return None since they will go through litellm.completion
|
| 715 |
+
"""
|
| 716 |
+
|
| 717 |
+
#########################################################
|
| 718 |
+
# Converse routes should go through litellm.completion()
|
| 719 |
+
if BedrockModelInfo._explicit_converse_route(model):
|
| 720 |
+
return None
|
| 721 |
+
|
| 722 |
+
#########################################################
|
| 723 |
+
# This goes through litellm.AmazonAnthropicClaude3MessagesConfig()
|
| 724 |
+
# Since bedrock Invoke supports Native Anthropic Messages API
|
| 725 |
+
#########################################################
|
| 726 |
+
if "claude" in model:
|
| 727 |
+
return litellm.AmazonAnthropicClaudeMessagesConfig()
|
| 728 |
+
|
| 729 |
+
#########################################################
|
| 730 |
+
# These routes will go through litellm.completion()
|
| 731 |
+
#########################################################
|
| 732 |
+
return None
|
| 733 |
+
|
| 734 |
+
|
| 735 |
+
def get_bedrock_chat_config(model: str):
|
| 736 |
+
"""
|
| 737 |
+
Helper function to get the appropriate Bedrock chat config based on model and route.
|
| 738 |
+
|
| 739 |
+
Args:
|
| 740 |
+
model: The model name/identifier
|
| 741 |
+
|
| 742 |
+
Returns:
|
| 743 |
+
The appropriate Bedrock config class instance
|
| 744 |
+
"""
|
| 745 |
+
bedrock_route = BedrockModelInfo.get_bedrock_route(model)
|
| 746 |
+
bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider(
|
| 747 |
+
model=model
|
| 748 |
+
)
|
| 749 |
+
base_model = BedrockModelInfo.get_base_model(model)
|
| 750 |
+
|
| 751 |
+
# Handle explicit routes first
|
| 752 |
+
if bedrock_route == "converse" or bedrock_route == "converse_like":
|
| 753 |
+
return litellm.AmazonConverseConfig()
|
| 754 |
+
elif bedrock_route == "openai":
|
| 755 |
+
return litellm.AmazonBedrockOpenAIConfig()
|
| 756 |
+
elif bedrock_route == "agent":
|
| 757 |
+
from litellm.llms.bedrock.chat.invoke_agent.transformation import (
|
| 758 |
+
AmazonInvokeAgentConfig,
|
| 759 |
+
)
|
| 760 |
+
|
| 761 |
+
return AmazonInvokeAgentConfig()
|
| 762 |
+
elif bedrock_route == "agentcore":
|
| 763 |
+
from litellm.llms.bedrock.chat.agentcore.transformation import (
|
| 764 |
+
AmazonAgentCoreConfig,
|
| 765 |
+
)
|
| 766 |
+
|
| 767 |
+
return AmazonAgentCoreConfig()
|
| 768 |
+
|
| 769 |
+
# Handle provider-specific configs
|
| 770 |
+
if bedrock_invoke_provider == "amazon":
|
| 771 |
+
return litellm.AmazonTitanConfig()
|
| 772 |
+
elif bedrock_invoke_provider == "anthropic":
|
| 773 |
+
if (
|
| 774 |
+
base_model
|
| 775 |
+
in litellm.AmazonAnthropicConfig.get_legacy_anthropic_model_names()
|
| 776 |
+
):
|
| 777 |
+
return litellm.AmazonAnthropicConfig()
|
| 778 |
+
else:
|
| 779 |
+
return litellm.AmazonAnthropicClaudeConfig()
|
| 780 |
+
elif bedrock_invoke_provider == "meta" or bedrock_invoke_provider == "llama":
|
| 781 |
+
return litellm.AmazonLlamaConfig()
|
| 782 |
+
elif bedrock_invoke_provider == "ai21":
|
| 783 |
+
return litellm.AmazonAI21Config()
|
| 784 |
+
elif bedrock_invoke_provider == "cohere":
|
| 785 |
+
return litellm.AmazonCohereConfig()
|
| 786 |
+
elif bedrock_invoke_provider == "mistral":
|
| 787 |
+
return litellm.AmazonMistralConfig()
|
| 788 |
+
elif bedrock_invoke_provider == "moonshot":
|
| 789 |
+
return litellm.AmazonMoonshotConfig()
|
| 790 |
+
elif bedrock_invoke_provider == "deepseek_r1":
|
| 791 |
+
return litellm.AmazonDeepSeekR1Config()
|
| 792 |
+
elif bedrock_invoke_provider == "nova":
|
| 793 |
+
return litellm.AmazonInvokeNovaConfig()
|
| 794 |
+
elif bedrock_invoke_provider == "qwen3":
|
| 795 |
+
return litellm.AmazonQwen3Config()
|
| 796 |
+
elif bedrock_invoke_provider == "qwen2":
|
| 797 |
+
return litellm.AmazonQwen2Config()
|
| 798 |
+
elif bedrock_invoke_provider == "twelvelabs":
|
| 799 |
+
return litellm.AmazonTwelveLabsPegasusConfig()
|
| 800 |
+
else:
|
| 801 |
+
return litellm.AmazonInvokeConfig()
|
| 802 |
+
|
| 803 |
+
|
| 804 |
+
class BedrockEventStreamDecoderBase:
|
| 805 |
+
"""
|
| 806 |
+
Base class for event stream decoding for Bedrock
|
| 807 |
+
"""
|
| 808 |
+
|
| 809 |
+
_response_stream_shape_cache = None
|
| 810 |
+
|
| 811 |
+
def __init__(self):
|
| 812 |
+
from botocore.parsers import EventStreamJSONParser
|
| 813 |
+
|
| 814 |
+
self.parser = EventStreamJSONParser()
|
| 815 |
+
|
| 816 |
+
def get_response_stream_shape(self):
|
| 817 |
+
if self._response_stream_shape_cache is None:
|
| 818 |
+
from botocore.loaders import Loader
|
| 819 |
+
from botocore.model import ServiceModel
|
| 820 |
+
|
| 821 |
+
loader = Loader()
|
| 822 |
+
bedrock_service_dict = loader.load_service_model(
|
| 823 |
+
"bedrock-runtime", "service-2"
|
| 824 |
+
)
|
| 825 |
+
bedrock_service_model = ServiceModel(bedrock_service_dict)
|
| 826 |
+
self._response_stream_shape_cache = bedrock_service_model.shape_for(
|
| 827 |
+
"ResponseStream"
|
| 828 |
+
)
|
| 829 |
+
|
| 830 |
+
return self._response_stream_shape_cache
|
| 831 |
+
|
| 832 |
+
def _parse_message_from_event(self, event) -> Optional[str]:
|
| 833 |
+
response_dict = event.to_response_dict()
|
| 834 |
+
parsed_response = self.parser.parse(
|
| 835 |
+
response_dict, self.get_response_stream_shape()
|
| 836 |
+
)
|
| 837 |
+
|
| 838 |
+
if response_dict["status_code"] != 200:
|
| 839 |
+
decoded_body = response_dict["body"].decode()
|
| 840 |
+
if isinstance(decoded_body, dict):
|
| 841 |
+
error_message = decoded_body.get("message")
|
| 842 |
+
elif isinstance(decoded_body, str):
|
| 843 |
+
error_message = decoded_body
|
| 844 |
+
else:
|
| 845 |
+
error_message = ""
|
| 846 |
+
exception_status = response_dict["headers"].get(":exception-type")
|
| 847 |
+
error_message = exception_status + " " + error_message
|
| 848 |
+
raise BedrockError(
|
| 849 |
+
status_code=response_dict["status_code"],
|
| 850 |
+
message=(
|
| 851 |
+
json.dumps(error_message)
|
| 852 |
+
if isinstance(error_message, dict)
|
| 853 |
+
else error_message
|
| 854 |
+
),
|
| 855 |
+
)
|
| 856 |
+
if "chunk" in parsed_response:
|
| 857 |
+
chunk = parsed_response.get("chunk")
|
| 858 |
+
if not chunk:
|
| 859 |
+
return None
|
| 860 |
+
return chunk.get("bytes").decode() # type: ignore[no-any-return]
|
| 861 |
+
else:
|
| 862 |
+
chunk = response_dict.get("body")
|
| 863 |
+
if not chunk:
|
| 864 |
+
return None
|
| 865 |
+
|
| 866 |
+
return chunk.decode() # type: ignore[no-any-return]
|
| 867 |
+
|
| 868 |
+
|
| 869 |
+
def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
|
| 870 |
+
"""
|
| 871 |
+
Extract anthropic-beta header values and convert them to a list.
|
| 872 |
+
Supports both JSON array format and comma-separated values from user headers.
|
| 873 |
+
|
| 874 |
+
Used by both converse and invoke transformations for consistent handling
|
| 875 |
+
of anthropic-beta headers that should be passed to AWS Bedrock.
|
| 876 |
+
|
| 877 |
+
Args:
|
| 878 |
+
headers (dict): Request headers dictionary
|
| 879 |
+
|
| 880 |
+
Returns:
|
| 881 |
+
List[str]: List of anthropic beta feature strings, empty list if no header
|
| 882 |
+
"""
|
| 883 |
+
anthropic_beta_header = headers.get("anthropic-beta")
|
| 884 |
+
if not anthropic_beta_header:
|
| 885 |
+
return []
|
| 886 |
+
|
| 887 |
+
# If it's already a list, return it
|
| 888 |
+
if isinstance(anthropic_beta_header, list):
|
| 889 |
+
return anthropic_beta_header
|
| 890 |
+
|
| 891 |
+
# Try to parse as JSON array first (e.g., '["interleaved-thinking-2025-05-14", "claude-code-20250219"]')
|
| 892 |
+
if isinstance(anthropic_beta_header, str):
|
| 893 |
+
anthropic_beta_header = anthropic_beta_header.strip()
|
| 894 |
+
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith(
|
| 895 |
+
"]"
|
| 896 |
+
):
|
| 897 |
+
try:
|
| 898 |
+
parsed = json.loads(anthropic_beta_header)
|
| 899 |
+
if isinstance(parsed, list):
|
| 900 |
+
return [str(beta).strip() for beta in parsed]
|
| 901 |
+
except json.JSONDecodeError:
|
| 902 |
+
pass # Fall through to comma-separated parsing
|
| 903 |
+
|
| 904 |
+
# Fall back to comma-separated values
|
| 905 |
+
return [beta.strip() for beta in anthropic_beta_header.split(",")]
|
| 906 |
+
|
| 907 |
+
return []
|
| 908 |
+
|
| 909 |
+
|
| 910 |
+
class CommonBatchFilesUtils:
|
| 911 |
+
"""
|
| 912 |
+
Common utilities for Bedrock batch and file operations.
|
| 913 |
+
Provides shared functionality to reduce code duplication between batches and files.
|
| 914 |
+
"""
|
| 915 |
+
|
| 916 |
+
def __init__(self):
|
| 917 |
+
# Import here to avoid circular imports
|
| 918 |
+
from .base_aws_llm import BaseAWSLLM
|
| 919 |
+
|
| 920 |
+
self._base_aws = BaseAWSLLM()
|
| 921 |
+
|
| 922 |
+
def get_bedrock_model_id_from_litellm_model(self, model: str) -> str:
|
| 923 |
+
"""
|
| 924 |
+
Extract the actual Bedrock model ID from LiteLLM model name.
|
| 925 |
+
|
| 926 |
+
Args:
|
| 927 |
+
model: LiteLLM model name (e.g., "bedrock/anthropic.claude-3-sonnet-20240229-v1:0")
|
| 928 |
+
|
| 929 |
+
Returns:
|
| 930 |
+
Bedrock model ID (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
|
| 931 |
+
"""
|
| 932 |
+
if model.startswith("bedrock/"):
|
| 933 |
+
return model[8:] # Remove "bedrock/" prefix
|
| 934 |
+
return model
|
| 935 |
+
|
| 936 |
+
def parse_s3_uri(self, s3_uri: str) -> tuple:
|
| 937 |
+
"""
|
| 938 |
+
Parse S3 URI into bucket and key components.
|
| 939 |
+
|
| 940 |
+
Args:
|
| 941 |
+
s3_uri: S3 URI (e.g., "s3://bucket/key/path")
|
| 942 |
+
|
| 943 |
+
Returns:
|
| 944 |
+
Tuple of (bucket, key)
|
| 945 |
+
|
| 946 |
+
Raises:
|
| 947 |
+
ValueError: If URI format is invalid
|
| 948 |
+
"""
|
| 949 |
+
if not s3_uri.startswith("s3://"):
|
| 950 |
+
raise ValueError(f"Invalid S3 URI format: {s3_uri}")
|
| 951 |
+
|
| 952 |
+
s3_parts = s3_uri[5:].split("/", 1) # Remove "s3://" and split on first "/"
|
| 953 |
+
if len(s3_parts) != 2:
|
| 954 |
+
raise ValueError(f"Invalid S3 URI format: {s3_uri}")
|
| 955 |
+
|
| 956 |
+
return s3_parts[0], s3_parts[1] # bucket, key
|
| 957 |
+
|
| 958 |
+
def extract_model_from_s3_file_path(
|
| 959 |
+
self, s3_uri: str, optional_params: dict
|
| 960 |
+
) -> str:
|
| 961 |
+
"""
|
| 962 |
+
Extract model ID from S3 file path.
|
| 963 |
+
|
| 964 |
+
The Bedrock file transformation creates S3 objects with the model name embedded:
|
| 965 |
+
Format: s3://bucket/litellm-bedrock-files-{model}-{uuid}.jsonl
|
| 966 |
+
"""
|
| 967 |
+
# Check if model is provided in optional_params first
|
| 968 |
+
if "model" in optional_params and optional_params["model"]:
|
| 969 |
+
return self.get_bedrock_model_id_from_litellm_model(
|
| 970 |
+
optional_params["model"]
|
| 971 |
+
)
|
| 972 |
+
|
| 973 |
+
# Extract model from S3 URI path
|
| 974 |
+
# Expected format: s3://bucket/litellm-bedrock-files-{model}-{uuid}.jsonl
|
| 975 |
+
try:
|
| 976 |
+
bucket, object_key = self.parse_s3_uri(s3_uri)
|
| 977 |
+
|
| 978 |
+
# Extract model from object key if it follows our naming pattern
|
| 979 |
+
if object_key.startswith("litellm-bedrock-files-"):
|
| 980 |
+
# Remove prefix and suffix to get model part
|
| 981 |
+
model_part = object_key[22:] # Remove "litellm-bedrock-files-"
|
| 982 |
+
# Find the last dash before the UUID
|
| 983 |
+
parts = model_part.split("-")
|
| 984 |
+
if len(parts) > 1:
|
| 985 |
+
# Reconstruct model name (everything except the last UUID part and .jsonl)
|
| 986 |
+
model_name = "-".join(parts[:-1])
|
| 987 |
+
if model_name.endswith(".jsonl"):
|
| 988 |
+
model_name = model_name[:-6] # Remove .jsonl
|
| 989 |
+
return model_name
|
| 990 |
+
except Exception:
|
| 991 |
+
pass
|
| 992 |
+
|
| 993 |
+
# Fallback to default model
|
| 994 |
+
return "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
| 995 |
+
|
| 996 |
+
def sign_aws_request(
|
| 997 |
+
self,
|
| 998 |
+
service_name: str,
|
| 999 |
+
data: Union[str, dict, "BedrockCreateBatchRequest"],
|
| 1000 |
+
endpoint_url: str,
|
| 1001 |
+
optional_params: dict,
|
| 1002 |
+
method: str = "POST",
|
| 1003 |
+
) -> tuple:
|
| 1004 |
+
"""
|
| 1005 |
+
Sign AWS request using Signature Version 4.
|
| 1006 |
+
|
| 1007 |
+
Args:
|
| 1008 |
+
service_name: AWS service name ("bedrock" or "s3")
|
| 1009 |
+
data: Request data (string or dict)
|
| 1010 |
+
endpoint_url: Full endpoint URL
|
| 1011 |
+
optional_params: Optional parameters containing AWS credentials
|
| 1012 |
+
method: HTTP method (default: POST)
|
| 1013 |
+
|
| 1014 |
+
Returns:
|
| 1015 |
+
Tuple of (signed_headers, signed_data)
|
| 1016 |
+
"""
|
| 1017 |
+
try:
|
| 1018 |
+
from botocore.auth import SigV4Auth
|
| 1019 |
+
from botocore.awsrequest import AWSRequest
|
| 1020 |
+
except ImportError:
|
| 1021 |
+
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
|
| 1022 |
+
|
| 1023 |
+
# Get AWS credentials using existing methods
|
| 1024 |
+
aws_region_name = self._base_aws._get_aws_region_name(
|
| 1025 |
+
optional_params=optional_params, model=""
|
| 1026 |
+
)
|
| 1027 |
+
credentials = self._base_aws.get_credentials(
|
| 1028 |
+
aws_access_key_id=optional_params.get("aws_access_key_id"),
|
| 1029 |
+
aws_secret_access_key=optional_params.get("aws_secret_access_key"),
|
| 1030 |
+
aws_session_token=optional_params.get("aws_session_token"),
|
| 1031 |
+
aws_region_name=aws_region_name,
|
| 1032 |
+
aws_session_name=optional_params.get("aws_session_name"),
|
| 1033 |
+
aws_profile_name=optional_params.get("aws_profile_name"),
|
| 1034 |
+
aws_role_name=optional_params.get("aws_role_name"),
|
| 1035 |
+
aws_web_identity_token=optional_params.get("aws_web_identity_token"),
|
| 1036 |
+
aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
|
| 1037 |
+
)
|
| 1038 |
+
|
| 1039 |
+
# Prepare the request data
|
| 1040 |
+
method_upper = method.upper()
|
| 1041 |
+
if method_upper == "GET":
|
| 1042 |
+
# GET requests should be signed with an empty payload
|
| 1043 |
+
request_data = ""
|
| 1044 |
+
headers = {}
|
| 1045 |
+
else:
|
| 1046 |
+
if isinstance(data, dict):
|
| 1047 |
+
import json
|
| 1048 |
+
|
| 1049 |
+
request_data = json.dumps(data)
|
| 1050 |
+
else:
|
| 1051 |
+
request_data = data
|
| 1052 |
+
# Prepare headers for non-GET requests
|
| 1053 |
+
headers = {"Content-Type": "application/json"}
|
| 1054 |
+
|
| 1055 |
+
# Create AWS request and sign it
|
| 1056 |
+
sigv4 = SigV4Auth(credentials, service_name, aws_region_name)
|
| 1057 |
+
request = AWSRequest(
|
| 1058 |
+
method=method_upper, url=endpoint_url, data=request_data, headers=headers
|
| 1059 |
+
)
|
| 1060 |
+
sigv4.add_auth(request)
|
| 1061 |
+
prepped = request.prepare()
|
| 1062 |
+
|
| 1063 |
+
return (
|
| 1064 |
+
dict(prepped.headers),
|
| 1065 |
+
request_data.encode("utf-8")
|
| 1066 |
+
if isinstance(request_data, str)
|
| 1067 |
+
else request_data,
|
| 1068 |
+
)
|
| 1069 |
+
|
| 1070 |
+
def generate_unique_job_name(self, model: str, prefix: str = "litellm") -> str:
|
| 1071 |
+
"""
|
| 1072 |
+
Generate a unique job name for AWS services.
|
| 1073 |
+
AWS services often have length limits, so this creates a concise name.
|
| 1074 |
+
|
| 1075 |
+
Args:
|
| 1076 |
+
model: Model name to include in the job name
|
| 1077 |
+
prefix: Prefix for the job name
|
| 1078 |
+
|
| 1079 |
+
Returns:
|
| 1080 |
+
Unique job name (≤ 63 characters for Bedrock compatibility)
|
| 1081 |
+
"""
|
| 1082 |
+
from litellm._uuid import uuid
|
| 1083 |
+
|
| 1084 |
+
unique_id = str(uuid.uuid4())[:8]
|
| 1085 |
+
# Format: {prefix}-batch-{model}-{uuid}
|
| 1086 |
+
# Example: litellm-batch-claude-266c398e
|
| 1087 |
+
job_name = f"{prefix}-batch-{unique_id}"
|
| 1088 |
+
|
| 1089 |
+
return job_name
|
| 1090 |
+
|
| 1091 |
+
def get_s3_bucket_and_key_from_config(
|
| 1092 |
+
self,
|
| 1093 |
+
litellm_params: dict,
|
| 1094 |
+
optional_params: dict,
|
| 1095 |
+
bucket_env_var: str = "AWS_S3_BUCKET_NAME",
|
| 1096 |
+
key_prefix: str = "litellm",
|
| 1097 |
+
) -> tuple:
|
| 1098 |
+
"""
|
| 1099 |
+
Get S3 bucket and generate a unique key from configuration.
|
| 1100 |
+
|
| 1101 |
+
Args:
|
| 1102 |
+
litellm_params: LiteLLM parameters
|
| 1103 |
+
optional_params: Optional parameters
|
| 1104 |
+
bucket_env_var: Environment variable name for bucket
|
| 1105 |
+
key_prefix: Prefix for the S3 key
|
| 1106 |
+
|
| 1107 |
+
Returns:
|
| 1108 |
+
Tuple of (bucket_name, object_key)
|
| 1109 |
+
"""
|
| 1110 |
+
import time
|
| 1111 |
+
|
| 1112 |
+
from litellm._uuid import uuid
|
| 1113 |
+
|
| 1114 |
+
# Get bucket name
|
| 1115 |
+
bucket_name = (
|
| 1116 |
+
litellm_params.get("s3_bucket_name")
|
| 1117 |
+
or optional_params.get("s3_bucket_name")
|
| 1118 |
+
or os.getenv(bucket_env_var)
|
| 1119 |
+
)
|
| 1120 |
+
if not bucket_name:
|
| 1121 |
+
raise ValueError(
|
| 1122 |
+
f"S3 bucket name is required. Set 's3_bucket_name' parameter or {bucket_env_var} env var"
|
| 1123 |
+
)
|
| 1124 |
+
|
| 1125 |
+
# Generate unique object key
|
| 1126 |
+
timestamp = int(time.time())
|
| 1127 |
+
unique_id = str(uuid.uuid4())[:8]
|
| 1128 |
+
object_key = f"{key_prefix}-{timestamp}-{unique_id}"
|
| 1129 |
+
|
| 1130 |
+
return bucket_name, object_key
|
| 1131 |
+
|
| 1132 |
+
def get_error_class(
|
| 1133 |
+
self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers]
|
| 1134 |
+
) -> BaseLLMException:
|
| 1135 |
+
"""
|
| 1136 |
+
Get Bedrock-specific error class.
|
| 1137 |
+
"""
|
| 1138 |
+
return BedrockError(
|
| 1139 |
+
status_code=status_code, message=error_message, headers=headers
|
| 1140 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/cost_calculation.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Helper util for handling bedrock-specific cost calculation
|
| 3 |
+
- e.g.: prompt caching
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from typing import TYPE_CHECKING, Optional, Tuple
|
| 7 |
+
|
| 8 |
+
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
| 9 |
+
|
| 10 |
+
if TYPE_CHECKING:
|
| 11 |
+
from litellm.types.utils import Usage
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def cost_per_token(
|
| 15 |
+
model: str, usage: "Usage", service_tier: Optional[str] = None
|
| 16 |
+
) -> Tuple[float, float]:
|
| 17 |
+
"""
|
| 18 |
+
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
|
| 19 |
+
|
| 20 |
+
Follows the same logic as Anthropic's cost per token calculation.
|
| 21 |
+
"""
|
| 22 |
+
return generic_cost_per_token(
|
| 23 |
+
model=model,
|
| 24 |
+
usage=usage,
|
| 25 |
+
custom_llm_provider="bedrock",
|
| 26 |
+
service_tier=service_tier,
|
| 27 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Bedrock Token Counter implementation using the CountTokens API.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import Any, Dict, List, Optional
|
| 6 |
+
|
| 7 |
+
from litellm._logging import verbose_logger
|
| 8 |
+
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
| 9 |
+
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
|
| 10 |
+
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
|
| 11 |
+
from litellm.types.utils import LlmProviders, TokenCountResponse
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class BedrockTokenCounter(BaseTokenCounter):
|
| 15 |
+
"""Token counter implementation for AWS Bedrock provider using the CountTokens API."""
|
| 16 |
+
|
| 17 |
+
def should_use_token_counting_api(
|
| 18 |
+
self,
|
| 19 |
+
custom_llm_provider: Optional[str] = None,
|
| 20 |
+
) -> bool:
|
| 21 |
+
"""
|
| 22 |
+
Returns True if we should use the Bedrock CountTokens API for token counting.
|
| 23 |
+
"""
|
| 24 |
+
return custom_llm_provider == LlmProviders.BEDROCK.value
|
| 25 |
+
|
| 26 |
+
async def count_tokens(
|
| 27 |
+
self,
|
| 28 |
+
model_to_use: str,
|
| 29 |
+
messages: Optional[List[Dict[str, Any]]],
|
| 30 |
+
contents: Optional[List[Dict[str, Any]]],
|
| 31 |
+
deployment: Optional[Dict[str, Any]] = None,
|
| 32 |
+
request_model: str = "",
|
| 33 |
+
tools: Optional[List[Dict[str, Any]]] = None,
|
| 34 |
+
system: Optional[Any] = None,
|
| 35 |
+
) -> Optional[TokenCountResponse]:
|
| 36 |
+
"""
|
| 37 |
+
Count tokens using AWS Bedrock's CountTokens API.
|
| 38 |
+
|
| 39 |
+
This method calls the existing BedrockCountTokensHandler to make an API call
|
| 40 |
+
to Bedrock's token counting endpoint, bypassing the local tiktoken-based counting.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
model_to_use: The model identifier
|
| 44 |
+
messages: The messages to count tokens for
|
| 45 |
+
contents: Alternative content format (not used for Bedrock)
|
| 46 |
+
deployment: Deployment configuration containing litellm_params
|
| 47 |
+
request_model: The original request model name
|
| 48 |
+
|
| 49 |
+
Returns:
|
| 50 |
+
TokenCountResponse with token count, or None if counting fails
|
| 51 |
+
"""
|
| 52 |
+
if not messages:
|
| 53 |
+
return None
|
| 54 |
+
|
| 55 |
+
deployment = deployment or {}
|
| 56 |
+
litellm_params = deployment.get("litellm_params", {})
|
| 57 |
+
|
| 58 |
+
# Build request data in the format expected by BedrockCountTokensHandler
|
| 59 |
+
request_data: Dict[str, Any] = {
|
| 60 |
+
"model": model_to_use,
|
| 61 |
+
"messages": messages,
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
if tools:
|
| 65 |
+
request_data["tools"] = tools
|
| 66 |
+
|
| 67 |
+
if system:
|
| 68 |
+
request_data["system"] = system
|
| 69 |
+
|
| 70 |
+
# Get the resolved model (strip prefixes like bedrock/, converse/, etc.)
|
| 71 |
+
resolved_model = get_bedrock_base_model(model_to_use)
|
| 72 |
+
|
| 73 |
+
try:
|
| 74 |
+
handler = BedrockCountTokensHandler()
|
| 75 |
+
result = await handler.handle_count_tokens_request(
|
| 76 |
+
request_data=request_data,
|
| 77 |
+
litellm_params=litellm_params,
|
| 78 |
+
resolved_model=resolved_model,
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# Transform response to TokenCountResponse
|
| 82 |
+
if result is not None:
|
| 83 |
+
return TokenCountResponse(
|
| 84 |
+
total_tokens=result.get("input_tokens", 0),
|
| 85 |
+
request_model=request_model,
|
| 86 |
+
model_used=model_to_use,
|
| 87 |
+
tokenizer_type="bedrock_api",
|
| 88 |
+
original_response=result,
|
| 89 |
+
)
|
| 90 |
+
except BedrockError as e:
|
| 91 |
+
verbose_logger.warning(
|
| 92 |
+
f"Bedrock CountTokens API error: status={e.status_code}, message={e.message}"
|
| 93 |
+
)
|
| 94 |
+
return TokenCountResponse(
|
| 95 |
+
total_tokens=0,
|
| 96 |
+
request_model=request_model,
|
| 97 |
+
model_used=model_to_use,
|
| 98 |
+
tokenizer_type="bedrock_api",
|
| 99 |
+
error=True,
|
| 100 |
+
error_message=e.message,
|
| 101 |
+
status_code=e.status_code,
|
| 102 |
+
)
|
| 103 |
+
except Exception as e:
|
| 104 |
+
verbose_logger.warning(f"Error calling Bedrock CountTokens API: {e}")
|
| 105 |
+
return TokenCountResponse(
|
| 106 |
+
total_tokens=0,
|
| 107 |
+
request_model=request_model,
|
| 108 |
+
model_used=model_to_use,
|
| 109 |
+
tokenizer_type="bedrock_api",
|
| 110 |
+
error=True,
|
| 111 |
+
error_message=str(e),
|
| 112 |
+
status_code=500,
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
return None
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/handler.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
AWS Bedrock CountTokens API handler.
|
| 3 |
+
|
| 4 |
+
Simplified handler leveraging existing LiteLLM Bedrock infrastructure.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Any, Dict
|
| 8 |
+
|
| 9 |
+
import httpx
|
| 10 |
+
|
| 11 |
+
import litellm
|
| 12 |
+
from litellm._logging import verbose_logger
|
| 13 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 14 |
+
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
|
| 15 |
+
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
| 19 |
+
"""
|
| 20 |
+
Simplified handler for AWS Bedrock CountTokens API requests.
|
| 21 |
+
|
| 22 |
+
Uses existing LiteLLM infrastructure for authentication and request handling.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
async def handle_count_tokens_request(
|
| 26 |
+
self,
|
| 27 |
+
request_data: Dict[str, Any],
|
| 28 |
+
litellm_params: Dict[str, Any],
|
| 29 |
+
resolved_model: str,
|
| 30 |
+
) -> Dict[str, Any]:
|
| 31 |
+
"""
|
| 32 |
+
Handle a CountTokens request using existing LiteLLM patterns.
|
| 33 |
+
|
| 34 |
+
Args:
|
| 35 |
+
request_data: The incoming request payload
|
| 36 |
+
litellm_params: LiteLLM configuration parameters
|
| 37 |
+
resolved_model: The actual model ID resolved from router
|
| 38 |
+
|
| 39 |
+
Returns:
|
| 40 |
+
Dictionary containing token count response
|
| 41 |
+
"""
|
| 42 |
+
try:
|
| 43 |
+
# Validate the request
|
| 44 |
+
self.validate_count_tokens_request(request_data)
|
| 45 |
+
|
| 46 |
+
verbose_logger.debug(
|
| 47 |
+
f"Processing CountTokens request for resolved model: {resolved_model}"
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
# Get AWS region using existing LiteLLM function
|
| 51 |
+
aws_region_name = self._get_aws_region_name(
|
| 52 |
+
optional_params=litellm_params,
|
| 53 |
+
model=resolved_model,
|
| 54 |
+
model_id=None,
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
verbose_logger.debug(f"Retrieved AWS region: {aws_region_name}")
|
| 58 |
+
|
| 59 |
+
# Transform request to Bedrock format (supports both Converse and InvokeModel)
|
| 60 |
+
bedrock_request = self.transform_anthropic_to_bedrock_count_tokens(
|
| 61 |
+
request_data=request_data
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
verbose_logger.debug(f"Transformed request: {bedrock_request}")
|
| 65 |
+
|
| 66 |
+
# Get endpoint URL using simplified function
|
| 67 |
+
api_base = litellm_params.get("api_base", None)
|
| 68 |
+
aws_bedrock_runtime_endpoint = litellm_params.get(
|
| 69 |
+
"aws_bedrock_runtime_endpoint", None
|
| 70 |
+
)
|
| 71 |
+
endpoint_url = self.get_bedrock_count_tokens_endpoint(
|
| 72 |
+
model=resolved_model,
|
| 73 |
+
aws_region_name=aws_region_name,
|
| 74 |
+
api_base=api_base,
|
| 75 |
+
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
verbose_logger.debug(f"Making request to: {endpoint_url}")
|
| 79 |
+
|
| 80 |
+
# Use existing _sign_request method from BaseAWSLLM
|
| 81 |
+
# Extract api_key for bearer token auth if provided
|
| 82 |
+
api_key = litellm_params.get("api_key", None)
|
| 83 |
+
headers = {"Content-Type": "application/json"}
|
| 84 |
+
signed_headers, signed_body = self._sign_request(
|
| 85 |
+
service_name="bedrock",
|
| 86 |
+
headers=headers,
|
| 87 |
+
optional_params=litellm_params,
|
| 88 |
+
request_data=bedrock_request,
|
| 89 |
+
api_base=endpoint_url,
|
| 90 |
+
model=resolved_model,
|
| 91 |
+
api_key=api_key,
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
async_client = get_async_httpx_client(
|
| 95 |
+
llm_provider=litellm.LlmProviders.BEDROCK
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
response = await async_client.post(
|
| 99 |
+
endpoint_url,
|
| 100 |
+
headers=signed_headers,
|
| 101 |
+
data=signed_body,
|
| 102 |
+
timeout=30.0,
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
verbose_logger.debug(f"Response status: {response.status_code}")
|
| 106 |
+
|
| 107 |
+
if response.status_code != 200:
|
| 108 |
+
error_text = response.text
|
| 109 |
+
verbose_logger.error(f"AWS Bedrock error: {error_text}")
|
| 110 |
+
raise BedrockError(
|
| 111 |
+
status_code=response.status_code,
|
| 112 |
+
message=error_text,
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
bedrock_response = response.json()
|
| 116 |
+
|
| 117 |
+
verbose_logger.debug(f"Bedrock response: {bedrock_response}")
|
| 118 |
+
|
| 119 |
+
# Transform response back to expected format
|
| 120 |
+
final_response = self.transform_bedrock_response_to_anthropic(
|
| 121 |
+
bedrock_response
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
verbose_logger.debug(f"Final response: {final_response}")
|
| 125 |
+
|
| 126 |
+
return final_response
|
| 127 |
+
|
| 128 |
+
except BedrockError:
|
| 129 |
+
# Re-raise Bedrock exceptions as-is
|
| 130 |
+
raise
|
| 131 |
+
except httpx.HTTPStatusError as e:
|
| 132 |
+
# HTTP errors - preserve the actual status code
|
| 133 |
+
verbose_logger.error(f"HTTP error in CountTokens handler: {str(e)}")
|
| 134 |
+
raise BedrockError(
|
| 135 |
+
status_code=e.response.status_code,
|
| 136 |
+
message=e.response.text,
|
| 137 |
+
)
|
| 138 |
+
except Exception as e:
|
| 139 |
+
verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
|
| 140 |
+
raise BedrockError(
|
| 141 |
+
status_code=500,
|
| 142 |
+
message=f"CountTokens processing error: {str(e)}",
|
| 143 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/transformation.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
AWS Bedrock CountTokens API transformation logic.
|
| 3 |
+
|
| 4 |
+
This module handles the transformation of requests from Anthropic Messages API format
|
| 5 |
+
to AWS Bedrock's CountTokens API format and vice versa.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import re
|
| 9 |
+
from typing import Any, Dict, List, Optional
|
| 10 |
+
|
| 11 |
+
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
| 12 |
+
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class BedrockCountTokensConfig(BaseAWSLLM):
|
| 16 |
+
"""
|
| 17 |
+
Configuration and transformation logic for AWS Bedrock CountTokens API.
|
| 18 |
+
|
| 19 |
+
AWS Bedrock CountTokens API Specification:
|
| 20 |
+
- Endpoint: POST /model/{modelId}/count-tokens
|
| 21 |
+
- Input formats: 'invokeModel' or 'converse'
|
| 22 |
+
- Response: {"inputTokens": <number>}
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
def _detect_input_type(self, request_data: Dict[str, Any]) -> str:
|
| 26 |
+
"""
|
| 27 |
+
Detect whether to use 'converse' or 'invokeModel' input format.
|
| 28 |
+
|
| 29 |
+
Args:
|
| 30 |
+
request_data: The original request data
|
| 31 |
+
|
| 32 |
+
Returns:
|
| 33 |
+
'converse' or 'invokeModel'
|
| 34 |
+
"""
|
| 35 |
+
# If the request has messages in the expected Anthropic format, use converse
|
| 36 |
+
if "messages" in request_data and isinstance(request_data["messages"], list):
|
| 37 |
+
return "converse"
|
| 38 |
+
|
| 39 |
+
# For raw text or other formats, use invokeModel
|
| 40 |
+
# This handles cases where the input is prompt-based or already in raw Bedrock format
|
| 41 |
+
return "invokeModel"
|
| 42 |
+
|
| 43 |
+
def transform_anthropic_to_bedrock_count_tokens(
|
| 44 |
+
self,
|
| 45 |
+
request_data: Dict[str, Any],
|
| 46 |
+
) -> Dict[str, Any]:
|
| 47 |
+
"""
|
| 48 |
+
Transform request to Bedrock CountTokens format.
|
| 49 |
+
Supports both Converse and InvokeModel input types.
|
| 50 |
+
|
| 51 |
+
Input (Anthropic format):
|
| 52 |
+
{
|
| 53 |
+
"model": "claude-3-5-sonnet",
|
| 54 |
+
"messages": [{"role": "user", "content": "Hello!"}]
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
Output (Bedrock CountTokens format for Converse):
|
| 58 |
+
{
|
| 59 |
+
"input": {
|
| 60 |
+
"converse": {
|
| 61 |
+
"messages": [...],
|
| 62 |
+
"system": [...] (if present)
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
Output (Bedrock CountTokens format for InvokeModel):
|
| 68 |
+
{
|
| 69 |
+
"input": {
|
| 70 |
+
"invokeModel": {
|
| 71 |
+
"body": "{...raw model input...}"
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
}
|
| 75 |
+
"""
|
| 76 |
+
input_type = self._detect_input_type(request_data)
|
| 77 |
+
|
| 78 |
+
if input_type == "converse":
|
| 79 |
+
return self._transform_to_converse_format(request_data)
|
| 80 |
+
else:
|
| 81 |
+
return self._transform_to_invoke_model_format(request_data)
|
| 82 |
+
|
| 83 |
+
def _transform_to_converse_format(
|
| 84 |
+
self, request_data: Dict[str, Any]
|
| 85 |
+
) -> Dict[str, Any]:
|
| 86 |
+
"""Transform to Converse input format, including system and tools."""
|
| 87 |
+
messages = request_data.get("messages", [])
|
| 88 |
+
system = request_data.get("system")
|
| 89 |
+
tools = request_data.get("tools")
|
| 90 |
+
|
| 91 |
+
# Transform messages
|
| 92 |
+
user_messages = []
|
| 93 |
+
for message in messages:
|
| 94 |
+
transformed_message: Dict[str, Any] = {
|
| 95 |
+
"role": message.get("role"),
|
| 96 |
+
"content": [],
|
| 97 |
+
}
|
| 98 |
+
content = message.get("content", "")
|
| 99 |
+
if isinstance(content, str):
|
| 100 |
+
transformed_message["content"].append({"text": content})
|
| 101 |
+
elif isinstance(content, list):
|
| 102 |
+
transformed_message["content"] = content
|
| 103 |
+
user_messages.append(transformed_message)
|
| 104 |
+
|
| 105 |
+
converse_input: Dict[str, Any] = {"messages": user_messages}
|
| 106 |
+
|
| 107 |
+
# Transform system prompt (string or list of blocks → Bedrock format)
|
| 108 |
+
system_blocks = self._transform_system(system)
|
| 109 |
+
if system_blocks:
|
| 110 |
+
converse_input["system"] = system_blocks
|
| 111 |
+
|
| 112 |
+
# Transform tools (Anthropic format → Bedrock toolConfig)
|
| 113 |
+
tool_config = self._transform_tools(tools)
|
| 114 |
+
if tool_config:
|
| 115 |
+
converse_input["toolConfig"] = tool_config
|
| 116 |
+
|
| 117 |
+
return {"input": {"converse": converse_input}}
|
| 118 |
+
|
| 119 |
+
def _transform_system(self, system: Optional[Any]) -> List[Dict[str, Any]]:
|
| 120 |
+
"""Transform Anthropic system prompt to Bedrock system blocks."""
|
| 121 |
+
if system is None:
|
| 122 |
+
return []
|
| 123 |
+
if isinstance(system, str):
|
| 124 |
+
return [{"text": system}]
|
| 125 |
+
if isinstance(system, list):
|
| 126 |
+
# Already in blocks format (e.g. [{"type": "text", "text": "..."}])
|
| 127 |
+
return [
|
| 128 |
+
{"text": block.get("text", "")}
|
| 129 |
+
for block in system
|
| 130 |
+
if isinstance(block, dict)
|
| 131 |
+
]
|
| 132 |
+
return []
|
| 133 |
+
|
| 134 |
+
def _transform_tools(
|
| 135 |
+
self, tools: Optional[List[Dict[str, Any]]]
|
| 136 |
+
) -> Optional[Dict[str, Any]]:
|
| 137 |
+
"""Transform Anthropic tools to Bedrock toolConfig format."""
|
| 138 |
+
if not tools:
|
| 139 |
+
return None
|
| 140 |
+
|
| 141 |
+
bedrock_tools = []
|
| 142 |
+
for tool in tools:
|
| 143 |
+
name = tool.get("name", "")
|
| 144 |
+
# Bedrock tool names must match [a-zA-Z][a-zA-Z0-9_]* and max 64 chars
|
| 145 |
+
name = re.sub(r"[^a-zA-Z0-9_]", "_", name)
|
| 146 |
+
if name and not name[0].isalpha():
|
| 147 |
+
name = "t_" + name
|
| 148 |
+
name = name[:64]
|
| 149 |
+
|
| 150 |
+
description = tool.get("description") or name
|
| 151 |
+
input_schema = tool.get(
|
| 152 |
+
"input_schema", {"type": "object", "properties": {}}
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
bedrock_tools.append(
|
| 156 |
+
{
|
| 157 |
+
"toolSpec": {
|
| 158 |
+
"name": name,
|
| 159 |
+
"description": description,
|
| 160 |
+
"inputSchema": {"json": input_schema},
|
| 161 |
+
}
|
| 162 |
+
}
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
return {"tools": bedrock_tools}
|
| 166 |
+
|
| 167 |
+
def _transform_to_invoke_model_format(
|
| 168 |
+
self, request_data: Dict[str, Any]
|
| 169 |
+
) -> Dict[str, Any]:
|
| 170 |
+
"""Transform to InvokeModel input format."""
|
| 171 |
+
import json
|
| 172 |
+
|
| 173 |
+
# For InvokeModel, we need to provide the raw body that would be sent to the model
|
| 174 |
+
# Remove the 'model' field from the body as it's not part of the model input
|
| 175 |
+
body_data = {k: v for k, v in request_data.items() if k != "model"}
|
| 176 |
+
|
| 177 |
+
return {"input": {"invokeModel": {"body": json.dumps(body_data)}}}
|
| 178 |
+
|
| 179 |
+
def get_bedrock_count_tokens_endpoint(
|
| 180 |
+
self,
|
| 181 |
+
model: str,
|
| 182 |
+
aws_region_name: str,
|
| 183 |
+
api_base: Optional[str] = None,
|
| 184 |
+
aws_bedrock_runtime_endpoint: Optional[str] = None,
|
| 185 |
+
) -> str:
|
| 186 |
+
"""
|
| 187 |
+
Construct the AWS Bedrock CountTokens API endpoint using existing LiteLLM functions.
|
| 188 |
+
|
| 189 |
+
Args:
|
| 190 |
+
model: The resolved model ID from router lookup
|
| 191 |
+
aws_region_name: AWS region (e.g., "eu-west-1")
|
| 192 |
+
api_base: Optional custom API base URL (takes highest priority)
|
| 193 |
+
aws_bedrock_runtime_endpoint: Optional custom Bedrock runtime endpoint
|
| 194 |
+
|
| 195 |
+
Returns:
|
| 196 |
+
Complete endpoint URL for CountTokens API
|
| 197 |
+
"""
|
| 198 |
+
# Use existing LiteLLM function to get the base model ID (removes region prefix)
|
| 199 |
+
model_id = get_bedrock_base_model(model)
|
| 200 |
+
|
| 201 |
+
# Remove bedrock/ prefix if present
|
| 202 |
+
if model_id.startswith("bedrock/"):
|
| 203 |
+
model_id = model_id[8:] # Remove "bedrock/" prefix
|
| 204 |
+
|
| 205 |
+
base_url, _ = self.get_runtime_endpoint(
|
| 206 |
+
api_base=api_base,
|
| 207 |
+
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
| 208 |
+
aws_region_name=aws_region_name,
|
| 209 |
+
)
|
| 210 |
+
endpoint = f"{base_url}/model/{model_id}/count-tokens"
|
| 211 |
+
|
| 212 |
+
return endpoint
|
| 213 |
+
|
| 214 |
+
def transform_bedrock_response_to_anthropic(
|
| 215 |
+
self, bedrock_response: Dict[str, Any]
|
| 216 |
+
) -> Dict[str, Any]:
|
| 217 |
+
"""
|
| 218 |
+
Transform Bedrock CountTokens response to Anthropic format.
|
| 219 |
+
|
| 220 |
+
Input (Bedrock response):
|
| 221 |
+
{
|
| 222 |
+
"inputTokens": 123
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
Output (Anthropic format):
|
| 226 |
+
{
|
| 227 |
+
"input_tokens": 123
|
| 228 |
+
}
|
| 229 |
+
"""
|
| 230 |
+
input_tokens = bedrock_response.get("inputTokens", 0)
|
| 231 |
+
|
| 232 |
+
return {"input_tokens": input_tokens}
|
| 233 |
+
|
| 234 |
+
def validate_count_tokens_request(self, request_data: Dict[str, Any]) -> None:
|
| 235 |
+
"""
|
| 236 |
+
Validate the incoming count tokens request.
|
| 237 |
+
Supports both Converse and InvokeModel input formats.
|
| 238 |
+
|
| 239 |
+
Args:
|
| 240 |
+
request_data: The request payload
|
| 241 |
+
|
| 242 |
+
Raises:
|
| 243 |
+
ValueError: If the request is invalid
|
| 244 |
+
"""
|
| 245 |
+
if not request_data.get("model"):
|
| 246 |
+
raise ValueError("model parameter is required")
|
| 247 |
+
|
| 248 |
+
input_type = self._detect_input_type(request_data)
|
| 249 |
+
|
| 250 |
+
if input_type == "converse":
|
| 251 |
+
# Validate Converse format (messages-based)
|
| 252 |
+
messages = request_data.get("messages", [])
|
| 253 |
+
if not messages:
|
| 254 |
+
raise ValueError("messages parameter is required for Converse input")
|
| 255 |
+
|
| 256 |
+
if not isinstance(messages, list):
|
| 257 |
+
raise ValueError("messages must be a list")
|
| 258 |
+
|
| 259 |
+
for i, message in enumerate(messages):
|
| 260 |
+
if not isinstance(message, dict):
|
| 261 |
+
raise ValueError(f"Message {i} must be a dictionary")
|
| 262 |
+
|
| 263 |
+
if "role" not in message:
|
| 264 |
+
raise ValueError(f"Message {i} must have a 'role' field")
|
| 265 |
+
|
| 266 |
+
if "content" not in message:
|
| 267 |
+
raise ValueError(f"Message {i} must have a 'content' field")
|
| 268 |
+
else:
|
| 269 |
+
# For InvokeModel format, we need at least some content to count tokens
|
| 270 |
+
# The content structure varies by model, so we do minimal validation
|
| 271 |
+
if len(request_data) <= 1: # Only has 'model' field
|
| 272 |
+
raise ValueError("Request must contain content to count tokens")
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_nova_transformation.py
ADDED
|
@@ -0,0 +1,361 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Nova /invoke and /async-invoke format.
|
| 3 |
+
|
| 4 |
+
Why separate file? Make it easy to see how transformation works
|
| 5 |
+
|
| 6 |
+
Supports:
|
| 7 |
+
- Synchronous embeddings (SINGLE_EMBEDDING)
|
| 8 |
+
- Asynchronous embeddings with segmentation (SEGMENTED_EMBEDDING)
|
| 9 |
+
- Multimodal inputs: text, image, video, audio
|
| 10 |
+
- Multiple embedding purposes and dimensions
|
| 11 |
+
|
| 12 |
+
Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from typing import List, Optional
|
| 16 |
+
|
| 17 |
+
from litellm.types.utils import (
|
| 18 |
+
Embedding,
|
| 19 |
+
EmbeddingResponse,
|
| 20 |
+
PromptTokensDetailsWrapper,
|
| 21 |
+
Usage,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class AmazonNovaEmbeddingConfig:
|
| 26 |
+
"""
|
| 27 |
+
Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html
|
| 28 |
+
|
| 29 |
+
Amazon Nova Multimodal Embeddings supports:
|
| 30 |
+
- Text, image, video, and audio inputs
|
| 31 |
+
- Synchronous (InvokeModel) and asynchronous (StartAsyncInvoke) APIs
|
| 32 |
+
- Multiple embedding purposes and dimensions
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
def __init__(self) -> None:
|
| 36 |
+
pass
|
| 37 |
+
|
| 38 |
+
def get_supported_openai_params(self) -> List[str]:
|
| 39 |
+
return [
|
| 40 |
+
"dimensions",
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
def map_openai_params(
|
| 44 |
+
self, non_default_params: dict, optional_params: dict
|
| 45 |
+
) -> dict:
|
| 46 |
+
"""Map OpenAI-style parameters to Nova parameters."""
|
| 47 |
+
for k, v in non_default_params.items():
|
| 48 |
+
if k == "dimensions":
|
| 49 |
+
# Map OpenAI dimensions to Nova embedding_dimension
|
| 50 |
+
optional_params["embedding_dimension"] = v
|
| 51 |
+
elif k in self.get_supported_openai_params():
|
| 52 |
+
optional_params[k] = v
|
| 53 |
+
return optional_params
|
| 54 |
+
|
| 55 |
+
def _parse_data_url(self, data_url: str) -> tuple:
|
| 56 |
+
"""
|
| 57 |
+
Parse a data URL to extract the media type and base64 data.
|
| 58 |
+
|
| 59 |
+
Args:
|
| 60 |
+
data_url: Data URL in format: data:image/jpeg;base64,/9j/4AAQ...
|
| 61 |
+
|
| 62 |
+
Returns:
|
| 63 |
+
tuple: (media_type, base64_data)
|
| 64 |
+
media_type: e.g., "image/jpeg", "video/mp4", "audio/mpeg"
|
| 65 |
+
base64_data: The base64-encoded data without the prefix
|
| 66 |
+
"""
|
| 67 |
+
if not data_url.startswith("data:"):
|
| 68 |
+
raise ValueError(f"Invalid data URL format: {data_url[:50]}...")
|
| 69 |
+
|
| 70 |
+
# Split by comma to separate metadata from data
|
| 71 |
+
# Format: data:image/jpeg;base64,<base64_data>
|
| 72 |
+
if "," not in data_url:
|
| 73 |
+
raise ValueError(
|
| 74 |
+
f"Invalid data URL format (missing comma): {data_url[:50]}..."
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
metadata, base64_data = data_url.split(",", 1)
|
| 78 |
+
|
| 79 |
+
# Extract media type from metadata
|
| 80 |
+
# Remove 'data:' prefix and ';base64' suffix
|
| 81 |
+
metadata = metadata[5:] # Remove 'data:'
|
| 82 |
+
|
| 83 |
+
if ";" in metadata:
|
| 84 |
+
media_type = metadata.split(";")[0]
|
| 85 |
+
else:
|
| 86 |
+
media_type = metadata
|
| 87 |
+
|
| 88 |
+
return media_type, base64_data
|
| 89 |
+
|
| 90 |
+
def _transform_request(
|
| 91 |
+
self,
|
| 92 |
+
input: str,
|
| 93 |
+
inference_params: dict,
|
| 94 |
+
async_invoke_route: bool = False,
|
| 95 |
+
model_id: Optional[str] = None,
|
| 96 |
+
output_s3_uri: Optional[str] = None,
|
| 97 |
+
) -> dict:
|
| 98 |
+
"""
|
| 99 |
+
Transform OpenAI-style input to Nova format.
|
| 100 |
+
|
| 101 |
+
Only handles OpenAI params (dimensions). All other Nova-specific params
|
| 102 |
+
should be passed via inference_params and will be passed through as-is.
|
| 103 |
+
|
| 104 |
+
Args:
|
| 105 |
+
input: The input text or media reference
|
| 106 |
+
inference_params: Additional parameters (will be passed through)
|
| 107 |
+
async_invoke_route: Whether this is for async invoke
|
| 108 |
+
model_id: Model ID (for async invoke)
|
| 109 |
+
output_s3_uri: S3 URI for output (for async invoke)
|
| 110 |
+
|
| 111 |
+
Returns:
|
| 112 |
+
dict: Nova embedding request
|
| 113 |
+
"""
|
| 114 |
+
# Determine task type
|
| 115 |
+
task_type = "SEGMENTED_EMBEDDING" if async_invoke_route else "SINGLE_EMBEDDING"
|
| 116 |
+
|
| 117 |
+
# Build the base request structure
|
| 118 |
+
request: dict = {
|
| 119 |
+
"schemaVersion": "nova-multimodal-embed-v1",
|
| 120 |
+
"taskType": task_type,
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
# Start with inference_params (user-provided params)
|
| 124 |
+
embedding_params = inference_params.copy()
|
| 125 |
+
|
| 126 |
+
embedding_params.pop("output_s3_uri", None)
|
| 127 |
+
|
| 128 |
+
# Map OpenAI dimensions to embeddingDimension if provided
|
| 129 |
+
if "dimensions" in embedding_params:
|
| 130 |
+
embedding_params["embeddingDimension"] = embedding_params.pop("dimensions")
|
| 131 |
+
elif "embedding_dimension" in embedding_params:
|
| 132 |
+
embedding_params["embeddingDimension"] = embedding_params.pop(
|
| 133 |
+
"embedding_dimension"
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
# Add required embeddingPurpose if not provided (required by Nova API)
|
| 137 |
+
if "embeddingPurpose" not in embedding_params:
|
| 138 |
+
embedding_params["embeddingPurpose"] = "GENERIC_INDEX"
|
| 139 |
+
|
| 140 |
+
# Add required embeddingDimension if not provided (required by Nova API)
|
| 141 |
+
if "embeddingDimension" not in embedding_params:
|
| 142 |
+
embedding_params["embeddingDimension"] = 3072
|
| 143 |
+
|
| 144 |
+
# For text/media input, add basic structure if user hasn't provided text/image/video/audio
|
| 145 |
+
if (
|
| 146 |
+
"text" not in embedding_params
|
| 147 |
+
and "image" not in embedding_params
|
| 148 |
+
and "video" not in embedding_params
|
| 149 |
+
and "audio" not in embedding_params
|
| 150 |
+
):
|
| 151 |
+
# Check if input is a data URL (e.g., data:image/jpeg;base64,...)
|
| 152 |
+
if input.startswith("data:"):
|
| 153 |
+
# Parse the data URL to extract media type and base64 data
|
| 154 |
+
media_type, base64_data = self._parse_data_url(input)
|
| 155 |
+
|
| 156 |
+
if media_type.startswith("image/"):
|
| 157 |
+
# Extract image format from MIME type (e.g., image/jpeg -> jpeg)
|
| 158 |
+
image_format = media_type.split("/")[1].lower()
|
| 159 |
+
# Nova API expects specific formats
|
| 160 |
+
if image_format == "jpg":
|
| 161 |
+
image_format = "jpeg"
|
| 162 |
+
|
| 163 |
+
embedding_params["image"] = {
|
| 164 |
+
"format": image_format,
|
| 165 |
+
"source": {"bytes": base64_data},
|
| 166 |
+
}
|
| 167 |
+
elif media_type.startswith("video/"):
|
| 168 |
+
# Handle video data URLs
|
| 169 |
+
video_format = media_type.split("/")[1].lower()
|
| 170 |
+
embedding_params["video"] = {
|
| 171 |
+
"format": video_format,
|
| 172 |
+
"source": {"bytes": base64_data},
|
| 173 |
+
}
|
| 174 |
+
elif media_type.startswith("audio/"):
|
| 175 |
+
# Handle audio data URLs
|
| 176 |
+
audio_format = media_type.split("/")[1].lower()
|
| 177 |
+
embedding_params["audio"] = {
|
| 178 |
+
"format": audio_format,
|
| 179 |
+
"source": {"bytes": base64_data},
|
| 180 |
+
}
|
| 181 |
+
else:
|
| 182 |
+
# Fallback to text for unknown types
|
| 183 |
+
embedding_params["text"] = {"value": input, "truncationMode": "END"}
|
| 184 |
+
elif input.startswith("s3://"):
|
| 185 |
+
# S3 URL - default to text for now, user should specify modality
|
| 186 |
+
embedding_params["text"] = {
|
| 187 |
+
"source": {"s3Location": {"uri": input}},
|
| 188 |
+
"truncationMode": "END", # Required by Nova API
|
| 189 |
+
}
|
| 190 |
+
else:
|
| 191 |
+
# Plain text input
|
| 192 |
+
embedding_params["text"] = {
|
| 193 |
+
"value": input,
|
| 194 |
+
"truncationMode": "END", # Required by Nova API
|
| 195 |
+
}
|
| 196 |
+
|
| 197 |
+
# Set the embedding params in the request
|
| 198 |
+
if task_type == "SINGLE_EMBEDDING":
|
| 199 |
+
request["singleEmbeddingParams"] = embedding_params
|
| 200 |
+
else:
|
| 201 |
+
request["segmentedEmbeddingParams"] = embedding_params
|
| 202 |
+
|
| 203 |
+
# For async invoke, wrap in the async invoke format
|
| 204 |
+
if async_invoke_route and model_id:
|
| 205 |
+
return self._wrap_async_invoke_request(
|
| 206 |
+
model_input=request,
|
| 207 |
+
model_id=model_id,
|
| 208 |
+
output_s3_uri=output_s3_uri,
|
| 209 |
+
)
|
| 210 |
+
|
| 211 |
+
return request
|
| 212 |
+
|
| 213 |
+
def _wrap_async_invoke_request(
|
| 214 |
+
self,
|
| 215 |
+
model_input: dict,
|
| 216 |
+
model_id: str,
|
| 217 |
+
output_s3_uri: Optional[str] = None,
|
| 218 |
+
) -> dict:
|
| 219 |
+
"""
|
| 220 |
+
Wrap the transformed request in the AWS Bedrock async invoke format.
|
| 221 |
+
|
| 222 |
+
Args:
|
| 223 |
+
model_input: The transformed Nova embedding request
|
| 224 |
+
model_id: The model identifier (without async_invoke prefix)
|
| 225 |
+
output_s3_uri: S3 URI for output data config
|
| 226 |
+
|
| 227 |
+
Returns:
|
| 228 |
+
dict: The wrapped async invoke request
|
| 229 |
+
"""
|
| 230 |
+
import urllib.parse
|
| 231 |
+
|
| 232 |
+
# Clean the model ID
|
| 233 |
+
unquoted_model_id = urllib.parse.unquote(model_id)
|
| 234 |
+
if unquoted_model_id.startswith("async_invoke/"):
|
| 235 |
+
unquoted_model_id = unquoted_model_id.replace("async_invoke/", "")
|
| 236 |
+
|
| 237 |
+
# Validate that the S3 URI is not empty
|
| 238 |
+
if not output_s3_uri or output_s3_uri.strip() == "":
|
| 239 |
+
raise ValueError("output_s3_uri is required for async invoke requests")
|
| 240 |
+
|
| 241 |
+
return {
|
| 242 |
+
"modelId": unquoted_model_id,
|
| 243 |
+
"modelInput": model_input,
|
| 244 |
+
"outputDataConfig": {"s3OutputDataConfig": {"s3Uri": output_s3_uri}},
|
| 245 |
+
}
|
| 246 |
+
|
| 247 |
+
def _transform_response(
|
| 248 |
+
self,
|
| 249 |
+
response_list: List[dict],
|
| 250 |
+
model: str,
|
| 251 |
+
batch_data: Optional[List[dict]] = None,
|
| 252 |
+
) -> EmbeddingResponse:
|
| 253 |
+
"""
|
| 254 |
+
Transform Nova response to OpenAI format.
|
| 255 |
+
|
| 256 |
+
Nova response format:
|
| 257 |
+
{
|
| 258 |
+
"embeddings": [
|
| 259 |
+
{
|
| 260 |
+
"embeddingType": "TEXT" | "IMAGE" | "VIDEO" | "AUDIO" | "AUDIO_VIDEO_COMBINED",
|
| 261 |
+
"embedding": [0.1, 0.2, ...],
|
| 262 |
+
"truncatedCharLength": 100 # Optional, only for text
|
| 263 |
+
}
|
| 264 |
+
]
|
| 265 |
+
}
|
| 266 |
+
"""
|
| 267 |
+
embeddings: List[Embedding] = []
|
| 268 |
+
total_tokens = 0
|
| 269 |
+
|
| 270 |
+
for response in response_list:
|
| 271 |
+
# Nova response has an "embeddings" array
|
| 272 |
+
if "embeddings" in response and isinstance(response["embeddings"], list):
|
| 273 |
+
for item in response["embeddings"]:
|
| 274 |
+
if "embedding" in item:
|
| 275 |
+
embedding = Embedding(
|
| 276 |
+
embedding=item["embedding"],
|
| 277 |
+
index=len(embeddings),
|
| 278 |
+
object="embedding",
|
| 279 |
+
)
|
| 280 |
+
embeddings.append(embedding)
|
| 281 |
+
|
| 282 |
+
# Estimate token count
|
| 283 |
+
# For text, use truncatedCharLength if available
|
| 284 |
+
if "truncatedCharLength" in item:
|
| 285 |
+
total_tokens += item["truncatedCharLength"] // 4
|
| 286 |
+
else:
|
| 287 |
+
# Rough estimate based on embedding dimension
|
| 288 |
+
total_tokens += len(item["embedding"]) // 4
|
| 289 |
+
elif "embedding" in response:
|
| 290 |
+
# Direct embedding response (fallback)
|
| 291 |
+
embedding = Embedding(
|
| 292 |
+
embedding=response["embedding"],
|
| 293 |
+
index=len(embeddings),
|
| 294 |
+
object="embedding",
|
| 295 |
+
)
|
| 296 |
+
embeddings.append(embedding)
|
| 297 |
+
total_tokens += len(response["embedding"]) // 4
|
| 298 |
+
|
| 299 |
+
# Count images from original requests for cost calculation
|
| 300 |
+
image_count = 0
|
| 301 |
+
if batch_data:
|
| 302 |
+
for request_data in batch_data:
|
| 303 |
+
# Nova wraps params in singleEmbeddingParams or segmentedEmbeddingParams
|
| 304 |
+
params = request_data.get(
|
| 305 |
+
"singleEmbeddingParams",
|
| 306 |
+
request_data.get("segmentedEmbeddingParams", {}),
|
| 307 |
+
)
|
| 308 |
+
if "image" in params:
|
| 309 |
+
image_count += 1
|
| 310 |
+
|
| 311 |
+
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
|
| 312 |
+
if image_count > 0:
|
| 313 |
+
prompt_tokens_details = PromptTokensDetailsWrapper(
|
| 314 |
+
image_count=image_count,
|
| 315 |
+
)
|
| 316 |
+
|
| 317 |
+
usage = Usage(
|
| 318 |
+
prompt_tokens=total_tokens,
|
| 319 |
+
total_tokens=total_tokens,
|
| 320 |
+
prompt_tokens_details=prompt_tokens_details,
|
| 321 |
+
)
|
| 322 |
+
|
| 323 |
+
return EmbeddingResponse(data=embeddings, model=model, usage=usage)
|
| 324 |
+
|
| 325 |
+
def _transform_async_invoke_response(
|
| 326 |
+
self, response: dict, model: str
|
| 327 |
+
) -> EmbeddingResponse:
|
| 328 |
+
"""
|
| 329 |
+
Transform async invoke response (invocation ARN) to OpenAI format.
|
| 330 |
+
|
| 331 |
+
AWS async invoke returns:
|
| 332 |
+
{
|
| 333 |
+
"invocationArn": "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"
|
| 334 |
+
}
|
| 335 |
+
|
| 336 |
+
We transform this to a job-like embedding response with the ARN in hidden params.
|
| 337 |
+
"""
|
| 338 |
+
invocation_arn = response.get("invocationArn", "")
|
| 339 |
+
|
| 340 |
+
# Create a placeholder embedding object for the job
|
| 341 |
+
embedding = Embedding(
|
| 342 |
+
embedding=[], # Empty embedding for async jobs
|
| 343 |
+
index=0,
|
| 344 |
+
object="embedding",
|
| 345 |
+
)
|
| 346 |
+
|
| 347 |
+
# Create usage object (empty for async jobs)
|
| 348 |
+
usage = Usage(prompt_tokens=0, total_tokens=0)
|
| 349 |
+
|
| 350 |
+
# Create hidden params with job ID
|
| 351 |
+
from litellm.types.llms.base import HiddenParams
|
| 352 |
+
|
| 353 |
+
hidden_params = HiddenParams()
|
| 354 |
+
setattr(hidden_params, "_invocation_arn", invocation_arn)
|
| 355 |
+
|
| 356 |
+
return EmbeddingResponse(
|
| 357 |
+
data=[embedding],
|
| 358 |
+
model=model,
|
| 359 |
+
usage=usage,
|
| 360 |
+
hidden_params=hidden_params,
|
| 361 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Titan G1 /invoke format.
|
| 3 |
+
|
| 4 |
+
Why separate file? Make it easy to see how transformation works
|
| 5 |
+
|
| 6 |
+
Convers
|
| 7 |
+
- G1 request format
|
| 8 |
+
|
| 9 |
+
Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import types
|
| 13 |
+
from typing import List
|
| 14 |
+
|
| 15 |
+
from litellm.types.llms.bedrock import (
|
| 16 |
+
AmazonTitanG1EmbeddingRequest,
|
| 17 |
+
AmazonTitanG1EmbeddingResponse,
|
| 18 |
+
)
|
| 19 |
+
from litellm.types.utils import Embedding, EmbeddingResponse, Usage
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class AmazonTitanG1Config:
|
| 23 |
+
"""
|
| 24 |
+
Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
def __init__(
|
| 28 |
+
self,
|
| 29 |
+
) -> None:
|
| 30 |
+
locals_ = locals().copy()
|
| 31 |
+
for key, value in locals_.items():
|
| 32 |
+
if key != "self" and value is not None:
|
| 33 |
+
setattr(self.__class__, key, value)
|
| 34 |
+
|
| 35 |
+
@classmethod
|
| 36 |
+
def get_config(cls):
|
| 37 |
+
return {
|
| 38 |
+
k: v
|
| 39 |
+
for k, v in cls.__dict__.items()
|
| 40 |
+
if not k.startswith("__")
|
| 41 |
+
and not isinstance(
|
| 42 |
+
v,
|
| 43 |
+
(
|
| 44 |
+
types.FunctionType,
|
| 45 |
+
types.BuiltinFunctionType,
|
| 46 |
+
classmethod,
|
| 47 |
+
staticmethod,
|
| 48 |
+
),
|
| 49 |
+
)
|
| 50 |
+
and v is not None
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
def get_supported_openai_params(self) -> List[str]:
|
| 54 |
+
return []
|
| 55 |
+
|
| 56 |
+
def map_openai_params(
|
| 57 |
+
self, non_default_params: dict, optional_params: dict
|
| 58 |
+
) -> dict:
|
| 59 |
+
return optional_params
|
| 60 |
+
|
| 61 |
+
def _transform_request(
|
| 62 |
+
self, input: str, inference_params: dict
|
| 63 |
+
) -> AmazonTitanG1EmbeddingRequest:
|
| 64 |
+
return AmazonTitanG1EmbeddingRequest(inputText=input)
|
| 65 |
+
|
| 66 |
+
def _transform_response(
|
| 67 |
+
self, response_list: List[dict], model: str
|
| 68 |
+
) -> EmbeddingResponse:
|
| 69 |
+
total_prompt_tokens = 0
|
| 70 |
+
|
| 71 |
+
transformed_responses: List[Embedding] = []
|
| 72 |
+
for index, response in enumerate(response_list):
|
| 73 |
+
_parsed_response = AmazonTitanG1EmbeddingResponse(**response) # type: ignore
|
| 74 |
+
transformed_responses.append(
|
| 75 |
+
Embedding(
|
| 76 |
+
embedding=_parsed_response["embedding"],
|
| 77 |
+
index=index,
|
| 78 |
+
object="embedding",
|
| 79 |
+
)
|
| 80 |
+
)
|
| 81 |
+
total_prompt_tokens += _parsed_response["inputTextTokenCount"]
|
| 82 |
+
|
| 83 |
+
usage = Usage(
|
| 84 |
+
prompt_tokens=total_prompt_tokens,
|
| 85 |
+
completion_tokens=0,
|
| 86 |
+
total_tokens=total_prompt_tokens,
|
| 87 |
+
)
|
| 88 |
+
return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Titan multimodal /invoke format.
|
| 3 |
+
|
| 4 |
+
Why separate file? Make it easy to see how transformation works
|
| 5 |
+
|
| 6 |
+
Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-mm.html
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from typing import List, Optional
|
| 10 |
+
|
| 11 |
+
from litellm.types.llms.bedrock import (
|
| 12 |
+
AmazonTitanMultimodalEmbeddingConfig,
|
| 13 |
+
AmazonTitanMultimodalEmbeddingRequest,
|
| 14 |
+
AmazonTitanMultimodalEmbeddingResponse,
|
| 15 |
+
)
|
| 16 |
+
from litellm.types.utils import (
|
| 17 |
+
Embedding,
|
| 18 |
+
EmbeddingResponse,
|
| 19 |
+
PromptTokensDetailsWrapper,
|
| 20 |
+
Usage,
|
| 21 |
+
)
|
| 22 |
+
from litellm.utils import get_base64_str, is_base64_encoded
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class AmazonTitanMultimodalEmbeddingG1Config:
|
| 26 |
+
"""
|
| 27 |
+
Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-mm.html
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
def __init__(self) -> None:
|
| 31 |
+
pass
|
| 32 |
+
|
| 33 |
+
def get_supported_openai_params(self) -> List[str]:
|
| 34 |
+
return ["dimensions"]
|
| 35 |
+
|
| 36 |
+
def map_openai_params(
|
| 37 |
+
self, non_default_params: dict, optional_params: dict
|
| 38 |
+
) -> dict:
|
| 39 |
+
for k, v in non_default_params.items():
|
| 40 |
+
if k == "dimensions":
|
| 41 |
+
optional_params[
|
| 42 |
+
"embeddingConfig"
|
| 43 |
+
] = AmazonTitanMultimodalEmbeddingConfig(outputEmbeddingLength=v)
|
| 44 |
+
return optional_params
|
| 45 |
+
|
| 46 |
+
def _transform_request(
|
| 47 |
+
self, input: str, inference_params: dict
|
| 48 |
+
) -> AmazonTitanMultimodalEmbeddingRequest:
|
| 49 |
+
## check if b64 encoded str or not ##
|
| 50 |
+
is_encoded = is_base64_encoded(input)
|
| 51 |
+
if is_encoded: # check if string is b64 encoded image or not
|
| 52 |
+
b64_str = get_base64_str(input)
|
| 53 |
+
transformed_request = AmazonTitanMultimodalEmbeddingRequest(
|
| 54 |
+
inputImage=b64_str
|
| 55 |
+
)
|
| 56 |
+
else:
|
| 57 |
+
transformed_request = AmazonTitanMultimodalEmbeddingRequest(inputText=input)
|
| 58 |
+
|
| 59 |
+
for k, v in inference_params.items():
|
| 60 |
+
transformed_request[k] = v # type: ignore
|
| 61 |
+
return transformed_request
|
| 62 |
+
|
| 63 |
+
def _transform_response(
|
| 64 |
+
self,
|
| 65 |
+
response_list: List[dict],
|
| 66 |
+
model: str,
|
| 67 |
+
batch_data: Optional[List[dict]] = None,
|
| 68 |
+
) -> EmbeddingResponse:
|
| 69 |
+
total_prompt_tokens = 0
|
| 70 |
+
transformed_responses: List[Embedding] = []
|
| 71 |
+
for index, response in enumerate(response_list):
|
| 72 |
+
_parsed_response = AmazonTitanMultimodalEmbeddingResponse(**response) # type: ignore
|
| 73 |
+
transformed_responses.append(
|
| 74 |
+
Embedding(
|
| 75 |
+
embedding=_parsed_response["embedding"],
|
| 76 |
+
index=index,
|
| 77 |
+
object="embedding",
|
| 78 |
+
)
|
| 79 |
+
)
|
| 80 |
+
total_prompt_tokens += _parsed_response["inputTextTokenCount"]
|
| 81 |
+
|
| 82 |
+
# Count images from original requests for cost calculation
|
| 83 |
+
image_count = 0
|
| 84 |
+
if batch_data:
|
| 85 |
+
for request_data in batch_data:
|
| 86 |
+
if "inputImage" in request_data:
|
| 87 |
+
image_count += 1
|
| 88 |
+
|
| 89 |
+
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
|
| 90 |
+
if image_count > 0:
|
| 91 |
+
prompt_tokens_details = PromptTokensDetailsWrapper(
|
| 92 |
+
image_count=image_count,
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
usage = Usage(
|
| 96 |
+
prompt_tokens=total_prompt_tokens,
|
| 97 |
+
completion_tokens=0,
|
| 98 |
+
total_tokens=total_prompt_tokens,
|
| 99 |
+
prompt_tokens_details=prompt_tokens_details,
|
| 100 |
+
)
|
| 101 |
+
return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Titan V2 /invoke format.
|
| 3 |
+
|
| 4 |
+
Why separate file? Make it easy to see how transformation works
|
| 5 |
+
|
| 6 |
+
Convers
|
| 7 |
+
- v2 request format
|
| 8 |
+
|
| 9 |
+
Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import types
|
| 13 |
+
from typing import List, Optional, Union
|
| 14 |
+
|
| 15 |
+
from litellm.types.llms.bedrock import (
|
| 16 |
+
AmazonTitanV2EmbeddingRequest,
|
| 17 |
+
AmazonTitanV2EmbeddingResponse,
|
| 18 |
+
)
|
| 19 |
+
from litellm.types.utils import Embedding, EmbeddingResponse, Usage
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class AmazonTitanV2Config:
|
| 23 |
+
"""
|
| 24 |
+
Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html
|
| 25 |
+
|
| 26 |
+
normalize: boolean - flag indicating whether or not to normalize the output embeddings. Defaults to true
|
| 27 |
+
dimensions: int - The number of dimensions the output embeddings should have. The following values are accepted: 1024 (default), 512, 256.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
normalize: Optional[bool] = None
|
| 31 |
+
dimensions: Optional[int] = None
|
| 32 |
+
|
| 33 |
+
def __init__(
|
| 34 |
+
self, normalize: Optional[bool] = None, dimensions: Optional[int] = None
|
| 35 |
+
) -> None:
|
| 36 |
+
locals_ = locals().copy()
|
| 37 |
+
for key, value in locals_.items():
|
| 38 |
+
if key != "self" and value is not None:
|
| 39 |
+
setattr(self.__class__, key, value)
|
| 40 |
+
|
| 41 |
+
@classmethod
|
| 42 |
+
def get_config(cls):
|
| 43 |
+
return {
|
| 44 |
+
k: v
|
| 45 |
+
for k, v in cls.__dict__.items()
|
| 46 |
+
if not k.startswith("__")
|
| 47 |
+
and not isinstance(
|
| 48 |
+
v,
|
| 49 |
+
(
|
| 50 |
+
types.FunctionType,
|
| 51 |
+
types.BuiltinFunctionType,
|
| 52 |
+
classmethod,
|
| 53 |
+
staticmethod,
|
| 54 |
+
),
|
| 55 |
+
)
|
| 56 |
+
and v is not None
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
def get_supported_openai_params(self) -> List[str]:
|
| 60 |
+
return ["dimensions", "encoding_format"]
|
| 61 |
+
|
| 62 |
+
def map_openai_params(
|
| 63 |
+
self, non_default_params: dict, optional_params: dict
|
| 64 |
+
) -> dict:
|
| 65 |
+
for k, v in non_default_params.items():
|
| 66 |
+
if k == "dimensions":
|
| 67 |
+
optional_params["dimensions"] = v
|
| 68 |
+
elif k == "encoding_format":
|
| 69 |
+
# Map OpenAI encoding_format to AWS embeddingTypes
|
| 70 |
+
if v == "float":
|
| 71 |
+
optional_params["embeddingTypes"] = ["float"]
|
| 72 |
+
elif v == "base64":
|
| 73 |
+
# base64 maps to binary format in AWS
|
| 74 |
+
optional_params["embeddingTypes"] = ["binary"]
|
| 75 |
+
else:
|
| 76 |
+
# For any other encoding format, default to float
|
| 77 |
+
optional_params["embeddingTypes"] = ["float"]
|
| 78 |
+
return optional_params
|
| 79 |
+
|
| 80 |
+
def _transform_request(
|
| 81 |
+
self, input: str, inference_params: dict
|
| 82 |
+
) -> AmazonTitanV2EmbeddingRequest:
|
| 83 |
+
return AmazonTitanV2EmbeddingRequest(inputText=input, **inference_params) # type: ignore
|
| 84 |
+
|
| 85 |
+
def _transform_response(
|
| 86 |
+
self, response_list: List[dict], model: str
|
| 87 |
+
) -> EmbeddingResponse:
|
| 88 |
+
total_prompt_tokens = 0
|
| 89 |
+
|
| 90 |
+
transformed_responses: List[Embedding] = []
|
| 91 |
+
for index, response in enumerate(response_list):
|
| 92 |
+
_parsed_response = AmazonTitanV2EmbeddingResponse(**response) # type: ignore
|
| 93 |
+
|
| 94 |
+
# According to AWS docs, embeddingsByType is always present
|
| 95 |
+
# If binary was requested (encoding_format="base64"), use binary data
|
| 96 |
+
# Otherwise, use float data from embeddingsByType or fallback to embedding field
|
| 97 |
+
embedding_data: Union[List[float], List[int]]
|
| 98 |
+
|
| 99 |
+
if (
|
| 100 |
+
"embeddingsByType" in _parsed_response
|
| 101 |
+
and "binary" in _parsed_response["embeddingsByType"]
|
| 102 |
+
):
|
| 103 |
+
# Use binary data if available (for encoding_format="base64")
|
| 104 |
+
embedding_data = _parsed_response["embeddingsByType"]["binary"]
|
| 105 |
+
elif (
|
| 106 |
+
"embeddingsByType" in _parsed_response
|
| 107 |
+
and "float" in _parsed_response["embeddingsByType"]
|
| 108 |
+
):
|
| 109 |
+
# Use float data from embeddingsByType
|
| 110 |
+
embedding_data = _parsed_response["embeddingsByType"]["float"]
|
| 111 |
+
elif "embedding" in _parsed_response:
|
| 112 |
+
# Fallback to legacy embedding field
|
| 113 |
+
embedding_data = _parsed_response["embedding"]
|
| 114 |
+
else:
|
| 115 |
+
raise ValueError(f"No embedding data found in response: {response}")
|
| 116 |
+
|
| 117 |
+
transformed_responses.append(
|
| 118 |
+
Embedding(
|
| 119 |
+
embedding=embedding_data,
|
| 120 |
+
index=index,
|
| 121 |
+
object="embedding",
|
| 122 |
+
)
|
| 123 |
+
)
|
| 124 |
+
total_prompt_tokens += _parsed_response["inputTextTokenCount"]
|
| 125 |
+
|
| 126 |
+
usage = Usage(
|
| 127 |
+
prompt_tokens=total_prompt_tokens,
|
| 128 |
+
completion_tokens=0,
|
| 129 |
+
total_tokens=total_prompt_tokens,
|
| 130 |
+
)
|
| 131 |
+
return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/cohere_transformation.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic from OpenAI /v1/embeddings format to Bedrock Cohere /invoke format.
|
| 3 |
+
|
| 4 |
+
Why separate file? Make it easy to see how transformation works
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import List
|
| 8 |
+
|
| 9 |
+
from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig
|
| 10 |
+
from litellm.types.llms.bedrock import CohereEmbeddingRequest
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class BedrockCohereEmbeddingConfig:
|
| 14 |
+
def __init__(self) -> None:
|
| 15 |
+
pass
|
| 16 |
+
|
| 17 |
+
def get_supported_openai_params(self) -> List[str]:
|
| 18 |
+
return ["encoding_format", "dimensions"]
|
| 19 |
+
|
| 20 |
+
def map_openai_params(
|
| 21 |
+
self, non_default_params: dict, optional_params: dict
|
| 22 |
+
) -> dict:
|
| 23 |
+
for k, v in non_default_params.items():
|
| 24 |
+
if k == "encoding_format":
|
| 25 |
+
optional_params["embedding_types"] = v
|
| 26 |
+
elif k == "dimensions":
|
| 27 |
+
optional_params["output_dimension"] = v
|
| 28 |
+
return optional_params
|
| 29 |
+
|
| 30 |
+
def _is_v3_model(self, model: str) -> bool:
|
| 31 |
+
return "3" in model
|
| 32 |
+
|
| 33 |
+
def _transform_request(
|
| 34 |
+
self, model: str, input: List[str], inference_params: dict
|
| 35 |
+
) -> CohereEmbeddingRequest:
|
| 36 |
+
transformed_request = CohereEmbeddingConfig()._transform_request(
|
| 37 |
+
model, input, inference_params
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
new_transformed_request = CohereEmbeddingRequest(
|
| 41 |
+
input_type=transformed_request["input_type"],
|
| 42 |
+
)
|
| 43 |
+
for k in CohereEmbeddingRequest.__annotations__.keys():
|
| 44 |
+
if k in transformed_request:
|
| 45 |
+
new_transformed_request[k] = transformed_request[k] # type: ignore
|
| 46 |
+
|
| 47 |
+
return new_transformed_request
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/embedding.py
ADDED
|
@@ -0,0 +1,699 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Handles embedding calls to Bedrock's `/invoke` endpoint
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import copy
|
| 6 |
+
import json
|
| 7 |
+
import urllib.parse
|
| 8 |
+
from typing import Any, Callable, List, Optional, Tuple, Union, get_args
|
| 9 |
+
|
| 10 |
+
import httpx
|
| 11 |
+
|
| 12 |
+
import litellm
|
| 13 |
+
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
|
| 14 |
+
from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
|
| 15 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 16 |
+
AsyncHTTPHandler,
|
| 17 |
+
HTTPHandler,
|
| 18 |
+
_get_httpx_client,
|
| 19 |
+
get_async_httpx_client,
|
| 20 |
+
)
|
| 21 |
+
from litellm.secret_managers.main import get_secret
|
| 22 |
+
from litellm.types.llms.bedrock import (
|
| 23 |
+
AmazonEmbeddingRequest,
|
| 24 |
+
CohereEmbeddingRequest,
|
| 25 |
+
)
|
| 26 |
+
from litellm.types.utils import EmbeddingResponse, LlmProviders
|
| 27 |
+
|
| 28 |
+
from ..base_aws_llm import BaseAWSLLM
|
| 29 |
+
from ..common_utils import BedrockError
|
| 30 |
+
from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
|
| 31 |
+
from .amazon_titan_g1_transformation import AmazonTitanG1Config
|
| 32 |
+
from .amazon_titan_multimodal_transformation import (
|
| 33 |
+
AmazonTitanMultimodalEmbeddingG1Config,
|
| 34 |
+
)
|
| 35 |
+
from .amazon_titan_v2_transformation import AmazonTitanV2Config
|
| 36 |
+
from .cohere_transformation import BedrockCohereEmbeddingConfig
|
| 37 |
+
from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class BedrockEmbedding(BaseAWSLLM):
|
| 41 |
+
def _load_credentials(
|
| 42 |
+
self,
|
| 43 |
+
optional_params: dict,
|
| 44 |
+
) -> Tuple[Any, str]:
|
| 45 |
+
try:
|
| 46 |
+
from botocore.credentials import Credentials
|
| 47 |
+
except ImportError:
|
| 48 |
+
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
|
| 49 |
+
## CREDENTIALS ##
|
| 50 |
+
# pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
|
| 51 |
+
aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
|
| 52 |
+
aws_access_key_id = optional_params.pop("aws_access_key_id", None)
|
| 53 |
+
aws_session_token = optional_params.pop("aws_session_token", None)
|
| 54 |
+
aws_region_name = optional_params.pop("aws_region_name", None)
|
| 55 |
+
aws_role_name = optional_params.pop("aws_role_name", None)
|
| 56 |
+
aws_session_name = optional_params.pop("aws_session_name", None)
|
| 57 |
+
aws_profile_name = optional_params.pop("aws_profile_name", None)
|
| 58 |
+
aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
|
| 59 |
+
aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
|
| 60 |
+
|
| 61 |
+
### SET REGION NAME ###
|
| 62 |
+
if aws_region_name is None:
|
| 63 |
+
# check env #
|
| 64 |
+
litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
|
| 65 |
+
|
| 66 |
+
if litellm_aws_region_name is not None and isinstance(
|
| 67 |
+
litellm_aws_region_name, str
|
| 68 |
+
):
|
| 69 |
+
aws_region_name = litellm_aws_region_name
|
| 70 |
+
|
| 71 |
+
standard_aws_region_name = get_secret("AWS_REGION", None)
|
| 72 |
+
if standard_aws_region_name is not None and isinstance(
|
| 73 |
+
standard_aws_region_name, str
|
| 74 |
+
):
|
| 75 |
+
aws_region_name = standard_aws_region_name
|
| 76 |
+
|
| 77 |
+
if aws_region_name is None:
|
| 78 |
+
aws_region_name = "us-west-2"
|
| 79 |
+
|
| 80 |
+
credentials: Credentials = self.get_credentials( # type: ignore
|
| 81 |
+
aws_access_key_id=aws_access_key_id,
|
| 82 |
+
aws_secret_access_key=aws_secret_access_key,
|
| 83 |
+
aws_session_token=aws_session_token,
|
| 84 |
+
aws_region_name=aws_region_name,
|
| 85 |
+
aws_session_name=aws_session_name,
|
| 86 |
+
aws_profile_name=aws_profile_name,
|
| 87 |
+
aws_role_name=aws_role_name,
|
| 88 |
+
aws_web_identity_token=aws_web_identity_token,
|
| 89 |
+
aws_sts_endpoint=aws_sts_endpoint,
|
| 90 |
+
)
|
| 91 |
+
return credentials, aws_region_name
|
| 92 |
+
|
| 93 |
+
async def async_embeddings(self):
|
| 94 |
+
pass
|
| 95 |
+
|
| 96 |
+
def _make_sync_call(
|
| 97 |
+
self,
|
| 98 |
+
client: Optional[HTTPHandler],
|
| 99 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 100 |
+
api_base: str,
|
| 101 |
+
headers: dict,
|
| 102 |
+
data: dict,
|
| 103 |
+
) -> dict:
|
| 104 |
+
if client is None or not isinstance(client, HTTPHandler):
|
| 105 |
+
_params = {}
|
| 106 |
+
if timeout is not None:
|
| 107 |
+
if isinstance(timeout, float) or isinstance(timeout, int):
|
| 108 |
+
timeout = httpx.Timeout(timeout)
|
| 109 |
+
_params["timeout"] = timeout
|
| 110 |
+
client = _get_httpx_client(_params) # type: ignore
|
| 111 |
+
else:
|
| 112 |
+
client = client
|
| 113 |
+
try:
|
| 114 |
+
response = client.post(url=api_base, headers=headers, data=json.dumps(data)) # type: ignore
|
| 115 |
+
response.raise_for_status()
|
| 116 |
+
except httpx.HTTPStatusError as err:
|
| 117 |
+
error_code = err.response.status_code
|
| 118 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 119 |
+
except httpx.TimeoutException:
|
| 120 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 121 |
+
|
| 122 |
+
return response.json()
|
| 123 |
+
|
| 124 |
+
async def _make_async_call(
|
| 125 |
+
self,
|
| 126 |
+
client: Optional[AsyncHTTPHandler],
|
| 127 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 128 |
+
api_base: str,
|
| 129 |
+
headers: dict,
|
| 130 |
+
data: dict,
|
| 131 |
+
) -> dict:
|
| 132 |
+
if client is None or not isinstance(client, AsyncHTTPHandler):
|
| 133 |
+
_params = {}
|
| 134 |
+
if timeout is not None:
|
| 135 |
+
if isinstance(timeout, float) or isinstance(timeout, int):
|
| 136 |
+
timeout = httpx.Timeout(timeout)
|
| 137 |
+
_params["timeout"] = timeout
|
| 138 |
+
client = get_async_httpx_client(
|
| 139 |
+
params=_params, llm_provider=litellm.LlmProviders.BEDROCK
|
| 140 |
+
)
|
| 141 |
+
else:
|
| 142 |
+
client = client
|
| 143 |
+
|
| 144 |
+
try:
|
| 145 |
+
response = await client.post(url=api_base, headers=headers, data=json.dumps(data)) # type: ignore
|
| 146 |
+
response.raise_for_status()
|
| 147 |
+
except httpx.HTTPStatusError as err:
|
| 148 |
+
error_code = err.response.status_code
|
| 149 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 150 |
+
except httpx.TimeoutException:
|
| 151 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 152 |
+
|
| 153 |
+
return response.json()
|
| 154 |
+
|
| 155 |
+
def _transform_response(
|
| 156 |
+
self,
|
| 157 |
+
response_list: List[dict],
|
| 158 |
+
model: str,
|
| 159 |
+
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
|
| 160 |
+
is_async_invoke: Optional[bool] = False,
|
| 161 |
+
batch_data: Optional[List[dict]] = None,
|
| 162 |
+
) -> Optional[EmbeddingResponse]:
|
| 163 |
+
"""
|
| 164 |
+
Transforms the response from the Bedrock embedding provider to the OpenAI format.
|
| 165 |
+
"""
|
| 166 |
+
returned_response: Optional[EmbeddingResponse] = None
|
| 167 |
+
|
| 168 |
+
# Handle async invoke responses (single response with invocationArn)
|
| 169 |
+
if (
|
| 170 |
+
is_async_invoke
|
| 171 |
+
and len(response_list) == 1
|
| 172 |
+
and "invocationArn" in response_list[0]
|
| 173 |
+
):
|
| 174 |
+
if provider == "twelvelabs":
|
| 175 |
+
returned_response = (
|
| 176 |
+
TwelveLabsMarengoEmbeddingConfig()._transform_async_invoke_response(
|
| 177 |
+
response=response_list[0], model=model
|
| 178 |
+
)
|
| 179 |
+
)
|
| 180 |
+
elif provider == "nova":
|
| 181 |
+
returned_response = (
|
| 182 |
+
AmazonNovaEmbeddingConfig()._transform_async_invoke_response(
|
| 183 |
+
response=response_list[0], model=model
|
| 184 |
+
)
|
| 185 |
+
)
|
| 186 |
+
else:
|
| 187 |
+
# For other providers, create a generic async response
|
| 188 |
+
invocation_arn = response_list[0].get("invocationArn", "")
|
| 189 |
+
|
| 190 |
+
from litellm.types.utils import Embedding, Usage
|
| 191 |
+
|
| 192 |
+
embedding = Embedding(
|
| 193 |
+
embedding=[],
|
| 194 |
+
index=0,
|
| 195 |
+
object="embedding", # Must be literal "embedding"
|
| 196 |
+
)
|
| 197 |
+
usage = Usage(prompt_tokens=0, total_tokens=0)
|
| 198 |
+
|
| 199 |
+
# Create hidden params with job ID
|
| 200 |
+
from litellm.types.llms.base import HiddenParams
|
| 201 |
+
|
| 202 |
+
hidden_params = HiddenParams()
|
| 203 |
+
setattr(hidden_params, "_invocation_arn", invocation_arn)
|
| 204 |
+
|
| 205 |
+
returned_response = EmbeddingResponse(
|
| 206 |
+
data=[embedding],
|
| 207 |
+
model=model,
|
| 208 |
+
usage=usage,
|
| 209 |
+
hidden_params=hidden_params,
|
| 210 |
+
)
|
| 211 |
+
else:
|
| 212 |
+
# Handle regular invoke responses
|
| 213 |
+
if model == "amazon.titan-embed-image-v1":
|
| 214 |
+
returned_response = (
|
| 215 |
+
AmazonTitanMultimodalEmbeddingG1Config()._transform_response(
|
| 216 |
+
response_list=response_list, model=model, batch_data=batch_data
|
| 217 |
+
)
|
| 218 |
+
)
|
| 219 |
+
elif model == "amazon.titan-embed-text-v1":
|
| 220 |
+
returned_response = AmazonTitanG1Config()._transform_response(
|
| 221 |
+
response_list=response_list, model=model
|
| 222 |
+
)
|
| 223 |
+
elif model == "amazon.titan-embed-text-v2:0":
|
| 224 |
+
returned_response = AmazonTitanV2Config()._transform_response(
|
| 225 |
+
response_list=response_list, model=model
|
| 226 |
+
)
|
| 227 |
+
elif provider == "twelvelabs":
|
| 228 |
+
returned_response = (
|
| 229 |
+
TwelveLabsMarengoEmbeddingConfig()._transform_response(
|
| 230 |
+
response_list=response_list, model=model
|
| 231 |
+
)
|
| 232 |
+
)
|
| 233 |
+
elif provider == "nova":
|
| 234 |
+
returned_response = AmazonNovaEmbeddingConfig()._transform_response(
|
| 235 |
+
response_list=response_list, model=model, batch_data=batch_data
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
##########################################################
|
| 239 |
+
# Validate returned response
|
| 240 |
+
##########################################################
|
| 241 |
+
if returned_response is None:
|
| 242 |
+
raise Exception(
|
| 243 |
+
"Unable to map model response to known provider format. model={}".format(
|
| 244 |
+
model
|
| 245 |
+
)
|
| 246 |
+
)
|
| 247 |
+
return returned_response
|
| 248 |
+
|
| 249 |
+
def _single_func_embeddings(
|
| 250 |
+
self,
|
| 251 |
+
client: Optional[HTTPHandler],
|
| 252 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 253 |
+
batch_data: List[dict],
|
| 254 |
+
credentials: Any,
|
| 255 |
+
extra_headers: Optional[dict],
|
| 256 |
+
endpoint_url: str,
|
| 257 |
+
aws_region_name: str,
|
| 258 |
+
model: str,
|
| 259 |
+
logging_obj: Any,
|
| 260 |
+
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
|
| 261 |
+
api_key: Optional[str] = None,
|
| 262 |
+
is_async_invoke: Optional[bool] = False,
|
| 263 |
+
):
|
| 264 |
+
responses: List[dict] = []
|
| 265 |
+
for data in batch_data:
|
| 266 |
+
headers = {"Content-Type": "application/json"}
|
| 267 |
+
if extra_headers is not None:
|
| 268 |
+
headers = {"Content-Type": "application/json", **extra_headers}
|
| 269 |
+
|
| 270 |
+
prepped = self.get_request_headers( # type: ignore # type: ignore
|
| 271 |
+
credentials=credentials,
|
| 272 |
+
aws_region_name=aws_region_name,
|
| 273 |
+
extra_headers=extra_headers,
|
| 274 |
+
endpoint_url=endpoint_url,
|
| 275 |
+
data=json.dumps(data),
|
| 276 |
+
headers=headers,
|
| 277 |
+
api_key=api_key,
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
## LOGGING
|
| 281 |
+
logging_obj.pre_call(
|
| 282 |
+
input=data,
|
| 283 |
+
api_key="",
|
| 284 |
+
additional_args={
|
| 285 |
+
"complete_input_dict": data,
|
| 286 |
+
"api_base": prepped.url,
|
| 287 |
+
"headers": prepped.headers,
|
| 288 |
+
},
|
| 289 |
+
)
|
| 290 |
+
headers_for_request = (
|
| 291 |
+
dict(prepped.headers) if hasattr(prepped, "headers") else {}
|
| 292 |
+
)
|
| 293 |
+
response = self._make_sync_call(
|
| 294 |
+
client=client,
|
| 295 |
+
timeout=timeout,
|
| 296 |
+
api_base=prepped.url,
|
| 297 |
+
headers=headers_for_request,
|
| 298 |
+
data=data,
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
## LOGGING
|
| 302 |
+
logging_obj.post_call(
|
| 303 |
+
input=data,
|
| 304 |
+
api_key="",
|
| 305 |
+
original_response=response,
|
| 306 |
+
additional_args={"complete_input_dict": data},
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
responses.append(response)
|
| 310 |
+
|
| 311 |
+
return self._transform_response(
|
| 312 |
+
response_list=responses,
|
| 313 |
+
model=model,
|
| 314 |
+
provider=provider,
|
| 315 |
+
is_async_invoke=is_async_invoke,
|
| 316 |
+
batch_data=batch_data,
|
| 317 |
+
)
|
| 318 |
+
|
| 319 |
+
async def _async_single_func_embeddings(
|
| 320 |
+
self,
|
| 321 |
+
client: Optional[AsyncHTTPHandler],
|
| 322 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 323 |
+
batch_data: List[dict],
|
| 324 |
+
credentials: Any,
|
| 325 |
+
extra_headers: Optional[dict],
|
| 326 |
+
endpoint_url: str,
|
| 327 |
+
aws_region_name: str,
|
| 328 |
+
model: str,
|
| 329 |
+
logging_obj: Any,
|
| 330 |
+
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
|
| 331 |
+
api_key: Optional[str] = None,
|
| 332 |
+
is_async_invoke: Optional[bool] = False,
|
| 333 |
+
):
|
| 334 |
+
responses: List[dict] = []
|
| 335 |
+
for data in batch_data:
|
| 336 |
+
headers = {"Content-Type": "application/json"}
|
| 337 |
+
if extra_headers is not None:
|
| 338 |
+
headers = {"Content-Type": "application/json", **extra_headers}
|
| 339 |
+
|
| 340 |
+
prepped = self.get_request_headers( # type: ignore # type: ignore
|
| 341 |
+
credentials=credentials,
|
| 342 |
+
aws_region_name=aws_region_name,
|
| 343 |
+
extra_headers=extra_headers,
|
| 344 |
+
endpoint_url=endpoint_url,
|
| 345 |
+
data=json.dumps(data),
|
| 346 |
+
headers=headers,
|
| 347 |
+
api_key=api_key,
|
| 348 |
+
)
|
| 349 |
+
|
| 350 |
+
## LOGGING
|
| 351 |
+
logging_obj.pre_call(
|
| 352 |
+
input=data,
|
| 353 |
+
api_key="",
|
| 354 |
+
additional_args={
|
| 355 |
+
"complete_input_dict": data,
|
| 356 |
+
"api_base": prepped.url,
|
| 357 |
+
"headers": prepped.headers,
|
| 358 |
+
},
|
| 359 |
+
)
|
| 360 |
+
# Convert CaseInsensitiveDict to regular dict for httpx compatibility
|
| 361 |
+
# This ensures custom headers are properly forwarded, especially with IAM roles and custom api_base
|
| 362 |
+
headers_for_request = (
|
| 363 |
+
dict(prepped.headers) if hasattr(prepped, "headers") else {}
|
| 364 |
+
)
|
| 365 |
+
response = await self._make_async_call(
|
| 366 |
+
client=client,
|
| 367 |
+
timeout=timeout,
|
| 368 |
+
api_base=prepped.url,
|
| 369 |
+
headers=headers_for_request,
|
| 370 |
+
data=data,
|
| 371 |
+
)
|
| 372 |
+
|
| 373 |
+
## LOGGING
|
| 374 |
+
logging_obj.post_call(
|
| 375 |
+
input=data,
|
| 376 |
+
api_key="",
|
| 377 |
+
original_response=response,
|
| 378 |
+
additional_args={"complete_input_dict": data},
|
| 379 |
+
)
|
| 380 |
+
|
| 381 |
+
responses.append(response)
|
| 382 |
+
## TRANSFORM RESPONSE ##
|
| 383 |
+
return self._transform_response(
|
| 384 |
+
response_list=responses,
|
| 385 |
+
model=model,
|
| 386 |
+
provider=provider,
|
| 387 |
+
is_async_invoke=is_async_invoke,
|
| 388 |
+
batch_data=batch_data,
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
def embeddings( # noqa: PLR0915
|
| 392 |
+
self,
|
| 393 |
+
model: str,
|
| 394 |
+
input: List[str],
|
| 395 |
+
api_base: Optional[str],
|
| 396 |
+
model_response: EmbeddingResponse,
|
| 397 |
+
print_verbose: Callable,
|
| 398 |
+
encoding,
|
| 399 |
+
logging_obj,
|
| 400 |
+
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]],
|
| 401 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 402 |
+
aembedding: Optional[bool],
|
| 403 |
+
extra_headers: Optional[dict],
|
| 404 |
+
optional_params: dict,
|
| 405 |
+
litellm_params: dict,
|
| 406 |
+
api_key: Optional[str] = None,
|
| 407 |
+
) -> EmbeddingResponse:
|
| 408 |
+
credentials, aws_region_name = self._load_credentials(optional_params)
|
| 409 |
+
|
| 410 |
+
### TRANSFORMATION ###
|
| 411 |
+
unencoded_model_id = (
|
| 412 |
+
optional_params.pop("model_id", None) or model
|
| 413 |
+
) # default to model if not passed
|
| 414 |
+
modelId = urllib.parse.quote(unencoded_model_id, safe="")
|
| 415 |
+
aws_region_name = self._get_aws_region_name(
|
| 416 |
+
optional_params={"aws_region_name": aws_region_name},
|
| 417 |
+
model=model,
|
| 418 |
+
model_id=unencoded_model_id,
|
| 419 |
+
)
|
| 420 |
+
# Check async invoke needs to be used
|
| 421 |
+
has_async_invoke = "async_invoke/" in model
|
| 422 |
+
if has_async_invoke:
|
| 423 |
+
model = model.replace("async_invoke/", "", 1)
|
| 424 |
+
provider = self.get_bedrock_embedding_provider(model)
|
| 425 |
+
if provider is None:
|
| 426 |
+
raise Exception(
|
| 427 |
+
f"Unable to determine bedrock embedding provider for model: {model}. "
|
| 428 |
+
f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}"
|
| 429 |
+
)
|
| 430 |
+
inference_params = copy.deepcopy(optional_params)
|
| 431 |
+
inference_params = {
|
| 432 |
+
k: v
|
| 433 |
+
for k, v in inference_params.items()
|
| 434 |
+
if k.lower() not in self.aws_authentication_params
|
| 435 |
+
}
|
| 436 |
+
inference_params.pop(
|
| 437 |
+
"user", None
|
| 438 |
+
) # make sure user is not passed in for bedrock call
|
| 439 |
+
|
| 440 |
+
data: Optional[CohereEmbeddingRequest] = None
|
| 441 |
+
batch_data: Optional[List] = None
|
| 442 |
+
if provider == "cohere":
|
| 443 |
+
data = BedrockCohereEmbeddingConfig()._transform_request(
|
| 444 |
+
model=model, input=input, inference_params=inference_params
|
| 445 |
+
)
|
| 446 |
+
elif provider == "amazon" and model in [
|
| 447 |
+
"amazon.titan-embed-image-v1",
|
| 448 |
+
"amazon.titan-embed-text-v1",
|
| 449 |
+
"amazon.titan-embed-text-v2:0",
|
| 450 |
+
]:
|
| 451 |
+
batch_data = []
|
| 452 |
+
for i in input:
|
| 453 |
+
if model == "amazon.titan-embed-image-v1":
|
| 454 |
+
transformed_request: (
|
| 455 |
+
AmazonEmbeddingRequest
|
| 456 |
+
) = AmazonTitanMultimodalEmbeddingG1Config()._transform_request(
|
| 457 |
+
input=i, inference_params=inference_params
|
| 458 |
+
)
|
| 459 |
+
elif model == "amazon.titan-embed-text-v1":
|
| 460 |
+
transformed_request = AmazonTitanG1Config()._transform_request(
|
| 461 |
+
input=i, inference_params=inference_params
|
| 462 |
+
)
|
| 463 |
+
elif model == "amazon.titan-embed-text-v2:0":
|
| 464 |
+
transformed_request = AmazonTitanV2Config()._transform_request(
|
| 465 |
+
input=i, inference_params=inference_params
|
| 466 |
+
)
|
| 467 |
+
else:
|
| 468 |
+
raise Exception(
|
| 469 |
+
"Unmapped model. Received={}. Expected={}".format(
|
| 470 |
+
model,
|
| 471 |
+
[
|
| 472 |
+
"amazon.titan-embed-image-v1",
|
| 473 |
+
"amazon.titan-embed-text-v1",
|
| 474 |
+
"amazon.titan-embed-text-v2:0",
|
| 475 |
+
],
|
| 476 |
+
)
|
| 477 |
+
)
|
| 478 |
+
batch_data.append(transformed_request)
|
| 479 |
+
elif provider == "twelvelabs":
|
| 480 |
+
batch_data = []
|
| 481 |
+
for i in input:
|
| 482 |
+
twelvelabs_request = (
|
| 483 |
+
TwelveLabsMarengoEmbeddingConfig()._transform_request(
|
| 484 |
+
input=i,
|
| 485 |
+
inference_params=inference_params,
|
| 486 |
+
async_invoke_route=has_async_invoke,
|
| 487 |
+
model_id=modelId,
|
| 488 |
+
output_s3_uri=inference_params.get("output_s3_uri"),
|
| 489 |
+
)
|
| 490 |
+
)
|
| 491 |
+
batch_data.append(twelvelabs_request)
|
| 492 |
+
elif provider == "nova":
|
| 493 |
+
batch_data = []
|
| 494 |
+
for i in input:
|
| 495 |
+
nova_request = AmazonNovaEmbeddingConfig()._transform_request(
|
| 496 |
+
input=i,
|
| 497 |
+
inference_params=inference_params,
|
| 498 |
+
async_invoke_route=has_async_invoke,
|
| 499 |
+
model_id=modelId,
|
| 500 |
+
output_s3_uri=inference_params.get("output_s3_uri"),
|
| 501 |
+
)
|
| 502 |
+
batch_data.append(nova_request)
|
| 503 |
+
|
| 504 |
+
### SET RUNTIME ENDPOINT ###
|
| 505 |
+
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
|
| 506 |
+
api_base=api_base,
|
| 507 |
+
aws_bedrock_runtime_endpoint=optional_params.pop(
|
| 508 |
+
"aws_bedrock_runtime_endpoint", None
|
| 509 |
+
),
|
| 510 |
+
aws_region_name=aws_region_name,
|
| 511 |
+
)
|
| 512 |
+
if has_async_invoke:
|
| 513 |
+
endpoint_url = f"{endpoint_url}/async-invoke"
|
| 514 |
+
else:
|
| 515 |
+
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
|
| 516 |
+
|
| 517 |
+
if batch_data is not None:
|
| 518 |
+
if aembedding:
|
| 519 |
+
return self._async_single_func_embeddings( # type: ignore
|
| 520 |
+
client=(
|
| 521 |
+
client
|
| 522 |
+
if client is not None and isinstance(client, AsyncHTTPHandler)
|
| 523 |
+
else None
|
| 524 |
+
),
|
| 525 |
+
timeout=timeout,
|
| 526 |
+
batch_data=batch_data,
|
| 527 |
+
credentials=credentials,
|
| 528 |
+
extra_headers=extra_headers,
|
| 529 |
+
endpoint_url=endpoint_url,
|
| 530 |
+
aws_region_name=aws_region_name,
|
| 531 |
+
model=model,
|
| 532 |
+
logging_obj=logging_obj,
|
| 533 |
+
api_key=api_key,
|
| 534 |
+
provider=provider,
|
| 535 |
+
is_async_invoke=has_async_invoke,
|
| 536 |
+
)
|
| 537 |
+
returned_response = self._single_func_embeddings(
|
| 538 |
+
client=(
|
| 539 |
+
client
|
| 540 |
+
if client is not None and isinstance(client, HTTPHandler)
|
| 541 |
+
else None
|
| 542 |
+
),
|
| 543 |
+
timeout=timeout,
|
| 544 |
+
batch_data=batch_data,
|
| 545 |
+
credentials=credentials,
|
| 546 |
+
extra_headers=extra_headers,
|
| 547 |
+
endpoint_url=endpoint_url,
|
| 548 |
+
aws_region_name=aws_region_name,
|
| 549 |
+
model=model,
|
| 550 |
+
logging_obj=logging_obj,
|
| 551 |
+
api_key=api_key,
|
| 552 |
+
provider=provider,
|
| 553 |
+
is_async_invoke=has_async_invoke,
|
| 554 |
+
)
|
| 555 |
+
if returned_response is None:
|
| 556 |
+
raise Exception("Unable to map Bedrock request to provider")
|
| 557 |
+
return returned_response
|
| 558 |
+
elif data is None:
|
| 559 |
+
raise Exception("Unable to map Bedrock request to provider")
|
| 560 |
+
|
| 561 |
+
headers = {"Content-Type": "application/json"}
|
| 562 |
+
if extra_headers is not None:
|
| 563 |
+
headers = {"Content-Type": "application/json", **extra_headers}
|
| 564 |
+
|
| 565 |
+
prepped = self.get_request_headers( # type: ignore
|
| 566 |
+
credentials=credentials,
|
| 567 |
+
aws_region_name=aws_region_name,
|
| 568 |
+
extra_headers=extra_headers,
|
| 569 |
+
endpoint_url=endpoint_url,
|
| 570 |
+
data=json.dumps(data),
|
| 571 |
+
headers=headers,
|
| 572 |
+
api_key=api_key,
|
| 573 |
+
)
|
| 574 |
+
|
| 575 |
+
## ROUTING ##
|
| 576 |
+
# Convert CaseInsensitiveDict to regular dict for httpx compatibility
|
| 577 |
+
headers_for_request = (
|
| 578 |
+
dict(prepped.headers) if hasattr(prepped, "headers") else {}
|
| 579 |
+
)
|
| 580 |
+
return cohere_embedding(
|
| 581 |
+
model=model,
|
| 582 |
+
input=input,
|
| 583 |
+
model_response=model_response,
|
| 584 |
+
logging_obj=logging_obj,
|
| 585 |
+
optional_params=optional_params,
|
| 586 |
+
encoding=encoding,
|
| 587 |
+
data=data, # type: ignore
|
| 588 |
+
complete_api_base=prepped.url,
|
| 589 |
+
api_key=None,
|
| 590 |
+
aembedding=aembedding,
|
| 591 |
+
timeout=timeout,
|
| 592 |
+
client=client,
|
| 593 |
+
headers=headers_for_request,
|
| 594 |
+
)
|
| 595 |
+
|
| 596 |
+
async def _get_async_invoke_status(
|
| 597 |
+
self, invocation_arn: str, aws_region_name: str, logging_obj=None, **kwargs
|
| 598 |
+
) -> dict:
|
| 599 |
+
"""
|
| 600 |
+
Get the status of an async invoke job using the GetAsyncInvoke operation.
|
| 601 |
+
|
| 602 |
+
Args:
|
| 603 |
+
invocation_arn: The invocation ARN from the async invoke response
|
| 604 |
+
aws_region_name: AWS region name
|
| 605 |
+
**kwargs: Additional parameters (credentials, etc.)
|
| 606 |
+
|
| 607 |
+
Returns:
|
| 608 |
+
dict: Status response from AWS Bedrock
|
| 609 |
+
"""
|
| 610 |
+
|
| 611 |
+
# Get AWS credentials using the same method as other Bedrock methods
|
| 612 |
+
credentials, _ = self._load_credentials(kwargs)
|
| 613 |
+
|
| 614 |
+
# Get the runtime endpoint
|
| 615 |
+
endpoint_url, _ = self.get_runtime_endpoint(
|
| 616 |
+
api_base=None,
|
| 617 |
+
aws_bedrock_runtime_endpoint=kwargs.get("aws_bedrock_runtime_endpoint"),
|
| 618 |
+
aws_region_name=aws_region_name,
|
| 619 |
+
)
|
| 620 |
+
|
| 621 |
+
from urllib.parse import quote
|
| 622 |
+
|
| 623 |
+
# Encode the ARN for use in URL path
|
| 624 |
+
encoded_arn = quote(invocation_arn, safe="")
|
| 625 |
+
status_url = f"{endpoint_url.rstrip('/')}/async-invoke/{encoded_arn}"
|
| 626 |
+
|
| 627 |
+
# Prepare headers for GET request
|
| 628 |
+
headers = {"Content-Type": "application/json"}
|
| 629 |
+
|
| 630 |
+
# Use AWSRequest directly for GET requests (get_request_headers hardcodes POST)
|
| 631 |
+
try:
|
| 632 |
+
from botocore.auth import SigV4Auth
|
| 633 |
+
from botocore.awsrequest import AWSRequest
|
| 634 |
+
except ImportError:
|
| 635 |
+
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
|
| 636 |
+
|
| 637 |
+
# Create AWSRequest with GET method and encoded URL
|
| 638 |
+
request = AWSRequest(
|
| 639 |
+
method="GET",
|
| 640 |
+
url=status_url,
|
| 641 |
+
data=None, # GET request, no body
|
| 642 |
+
headers=headers,
|
| 643 |
+
)
|
| 644 |
+
|
| 645 |
+
# Sign the request - SigV4Auth will create canonical string from request URL
|
| 646 |
+
sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name)
|
| 647 |
+
sigv4.add_auth(request)
|
| 648 |
+
|
| 649 |
+
# Prepare the request
|
| 650 |
+
prepped = request.prepare()
|
| 651 |
+
|
| 652 |
+
# LOGGING
|
| 653 |
+
if logging_obj is not None:
|
| 654 |
+
# Create custom curl command for GET request
|
| 655 |
+
masked_headers = logging_obj._get_masked_headers(prepped.headers)
|
| 656 |
+
formatted_headers = " ".join(
|
| 657 |
+
[f"-H '{k}: {v}'" for k, v in masked_headers.items()]
|
| 658 |
+
)
|
| 659 |
+
custom_curl = "\n\nGET Request Sent from LiteLLM:\n"
|
| 660 |
+
custom_curl += "curl -X GET \\\n"
|
| 661 |
+
custom_curl += f"{prepped.url} \\\n"
|
| 662 |
+
custom_curl += f"{formatted_headers}\n"
|
| 663 |
+
|
| 664 |
+
logging_obj.pre_call(
|
| 665 |
+
input=invocation_arn,
|
| 666 |
+
api_key="",
|
| 667 |
+
additional_args={
|
| 668 |
+
"complete_input_dict": {"invocation_arn": invocation_arn},
|
| 669 |
+
"api_base": prepped.url,
|
| 670 |
+
"headers": prepped.headers,
|
| 671 |
+
"request_str": custom_curl, # Override with custom GET curl command
|
| 672 |
+
},
|
| 673 |
+
)
|
| 674 |
+
|
| 675 |
+
# Make the GET request
|
| 676 |
+
client = get_async_httpx_client(llm_provider=LlmProviders.BEDROCK)
|
| 677 |
+
response = await client.get(
|
| 678 |
+
url=prepped.url,
|
| 679 |
+
headers=prepped.headers,
|
| 680 |
+
)
|
| 681 |
+
|
| 682 |
+
# LOGGING
|
| 683 |
+
if logging_obj is not None:
|
| 684 |
+
logging_obj.post_call(
|
| 685 |
+
input=invocation_arn,
|
| 686 |
+
api_key="",
|
| 687 |
+
original_response=response,
|
| 688 |
+
additional_args={
|
| 689 |
+
"complete_input_dict": {"invocation_arn": invocation_arn}
|
| 690 |
+
},
|
| 691 |
+
)
|
| 692 |
+
|
| 693 |
+
# Parse response
|
| 694 |
+
if response.status_code == 200:
|
| 695 |
+
return response.json()
|
| 696 |
+
else:
|
| 697 |
+
raise Exception(
|
| 698 |
+
f"Failed to get async invoke status: {response.status_code} - {response.text}"
|
| 699 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic from OpenAI /v1/embeddings format to Bedrock TwelveLabs Marengo /invoke and /async-invoke format.
|
| 3 |
+
|
| 4 |
+
Why separate file? Make it easy to see how transformation works
|
| 5 |
+
|
| 6 |
+
Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from typing import List, Optional, Union, cast
|
| 10 |
+
|
| 11 |
+
from litellm.types.llms.bedrock import (
|
| 12 |
+
TWELVELABS_EMBEDDING_INPUT_TYPES,
|
| 13 |
+
TwelveLabsAsyncInvokeRequest,
|
| 14 |
+
TwelveLabsMarengoEmbeddingRequest,
|
| 15 |
+
TwelveLabsOutputDataConfig,
|
| 16 |
+
TwelveLabsS3Location,
|
| 17 |
+
TwelveLabsS3OutputDataConfig,
|
| 18 |
+
)
|
| 19 |
+
from litellm.types.utils import Embedding, EmbeddingResponse, Usage
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class TwelveLabsMarengoEmbeddingConfig:
|
| 23 |
+
"""
|
| 24 |
+
Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html
|
| 25 |
+
|
| 26 |
+
Supports text, image, video, and audio inputs.
|
| 27 |
+
- InvokeModel: text and image inputs
|
| 28 |
+
- StartAsyncInvoke: video, audio, image, and text inputs
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
def __init__(self) -> None:
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
def get_supported_openai_params(self) -> List[str]:
|
| 35 |
+
return [
|
| 36 |
+
"encoding_format",
|
| 37 |
+
"textTruncate",
|
| 38 |
+
"embeddingOption",
|
| 39 |
+
"startSec",
|
| 40 |
+
"lengthSec",
|
| 41 |
+
"useFixedLengthSec",
|
| 42 |
+
"minClipSec",
|
| 43 |
+
"input_type",
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
+
def map_openai_params(
|
| 47 |
+
self, non_default_params: dict, optional_params: dict
|
| 48 |
+
) -> dict:
|
| 49 |
+
for k, v in non_default_params.items():
|
| 50 |
+
if k == "encoding_format":
|
| 51 |
+
# TwelveLabs doesn't have encoding_format, but we can map it to embeddingOption
|
| 52 |
+
if v == "float":
|
| 53 |
+
optional_params["embeddingOption"] = ["visual-text", "visual-image"]
|
| 54 |
+
elif k == "textTruncate":
|
| 55 |
+
optional_params["textTruncate"] = v
|
| 56 |
+
elif k == "embeddingOption":
|
| 57 |
+
optional_params["embeddingOption"] = v
|
| 58 |
+
elif k == "input_type":
|
| 59 |
+
# Map input_type to inputType for Bedrock
|
| 60 |
+
optional_params["inputType"] = v
|
| 61 |
+
elif k in ["startSec", "lengthSec", "useFixedLengthSec", "minClipSec"]:
|
| 62 |
+
optional_params[k] = v
|
| 63 |
+
return optional_params
|
| 64 |
+
|
| 65 |
+
def _extract_bucket_owner_from_params(self, inference_params: dict) -> str:
|
| 66 |
+
"""
|
| 67 |
+
Extract bucket owner from inference parameters.
|
| 68 |
+
"""
|
| 69 |
+
return inference_params.get("bucketOwner", "")
|
| 70 |
+
|
| 71 |
+
def _is_s3_url(self, input: str) -> bool:
|
| 72 |
+
"""Check if input is an S3 URL."""
|
| 73 |
+
return input.startswith("s3://")
|
| 74 |
+
|
| 75 |
+
def _transform_request(
|
| 76 |
+
self,
|
| 77 |
+
input: str,
|
| 78 |
+
inference_params: dict,
|
| 79 |
+
async_invoke_route: bool = False,
|
| 80 |
+
model_id: Optional[str] = None,
|
| 81 |
+
output_s3_uri: Optional[str] = None,
|
| 82 |
+
) -> Union[TwelveLabsMarengoEmbeddingRequest, TwelveLabsAsyncInvokeRequest]:
|
| 83 |
+
"""
|
| 84 |
+
Transform OpenAI-style input to TwelveLabs Marengo format/async-invoke format.
|
| 85 |
+
|
| 86 |
+
Supports:
|
| 87 |
+
- Text inputs (for both invoke and async-invoke)
|
| 88 |
+
- Image inputs (for both invoke and async-invoke)
|
| 89 |
+
- Video inputs (async-invoke only)
|
| 90 |
+
- Audio inputs (async-invoke only)
|
| 91 |
+
- S3 URLs for all media types (async-invoke only)
|
| 92 |
+
"""
|
| 93 |
+
# Get input_type or default to "text"
|
| 94 |
+
input_type = cast(
|
| 95 |
+
TWELVELABS_EMBEDDING_INPUT_TYPES,
|
| 96 |
+
inference_params.get("inputType")
|
| 97 |
+
or inference_params.get("input_type")
|
| 98 |
+
or "text",
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
# Validate that async-invoke is used for video/audio
|
| 102 |
+
if input_type in ["video", "audio"] and not async_invoke_route:
|
| 103 |
+
raise ValueError(
|
| 104 |
+
f"Input type '{input_type}' requires async_invoke route. "
|
| 105 |
+
f"Use model format: 'bedrock/async_invoke/model_id'"
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
transformed_request: TwelveLabsMarengoEmbeddingRequest = {
|
| 109 |
+
"inputType": input_type
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
if input_type == "text":
|
| 113 |
+
transformed_request["inputText"] = input
|
| 114 |
+
# Set default textTruncate if not specified
|
| 115 |
+
if "textTruncate" not in inference_params:
|
| 116 |
+
transformed_request["textTruncate"] = "end"
|
| 117 |
+
|
| 118 |
+
elif input_type in ["image", "video", "audio"]:
|
| 119 |
+
if self._is_s3_url(input):
|
| 120 |
+
# S3 URL input
|
| 121 |
+
s3_location: TwelveLabsS3Location = {"uri": input}
|
| 122 |
+
bucket_owner = self._extract_bucket_owner_from_params(inference_params)
|
| 123 |
+
if bucket_owner:
|
| 124 |
+
s3_location["bucketOwner"] = bucket_owner
|
| 125 |
+
|
| 126 |
+
transformed_request["mediaSource"] = {"s3Location": s3_location}
|
| 127 |
+
else:
|
| 128 |
+
# Base64 encoded input
|
| 129 |
+
if input.startswith("data:"):
|
| 130 |
+
# Extract base64 data from data URL
|
| 131 |
+
b64_str = input.split(",", 1)[1] if "," in input else input
|
| 132 |
+
else:
|
| 133 |
+
# Direct base64 string
|
| 134 |
+
from litellm.utils import get_base64_str
|
| 135 |
+
|
| 136 |
+
b64_str = get_base64_str(input)
|
| 137 |
+
|
| 138 |
+
transformed_request["mediaSource"] = {"base64String": b64_str}
|
| 139 |
+
|
| 140 |
+
# Apply any additional inference parameters
|
| 141 |
+
for k, v in inference_params.items():
|
| 142 |
+
if k not in [
|
| 143 |
+
"inputType",
|
| 144 |
+
"input_type", # Exclude both camelCase and snake_case
|
| 145 |
+
"inputText",
|
| 146 |
+
"mediaSource",
|
| 147 |
+
"bucketOwner", # Don't include bucketOwner in the request
|
| 148 |
+
]: # Don't override core fields
|
| 149 |
+
transformed_request[k] = v # type: ignore
|
| 150 |
+
|
| 151 |
+
# If async invoke route, wrap in the async invoke format
|
| 152 |
+
if async_invoke_route and model_id:
|
| 153 |
+
return self._wrap_async_invoke_request(
|
| 154 |
+
model_input=transformed_request,
|
| 155 |
+
model_id=model_id,
|
| 156 |
+
output_s3_uri=output_s3_uri,
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
return transformed_request
|
| 160 |
+
|
| 161 |
+
def _wrap_async_invoke_request(
|
| 162 |
+
self,
|
| 163 |
+
model_input: TwelveLabsMarengoEmbeddingRequest,
|
| 164 |
+
model_id: str,
|
| 165 |
+
output_s3_uri: Optional[str] = None,
|
| 166 |
+
) -> TwelveLabsAsyncInvokeRequest:
|
| 167 |
+
"""
|
| 168 |
+
Wrap the transformed request in the correct AWS Bedrock async invoke format.
|
| 169 |
+
|
| 170 |
+
Args:
|
| 171 |
+
model_input: The transformed TwelveLabs Marengo embedding request
|
| 172 |
+
model_id: The model identifier (without async_invoke prefix)
|
| 173 |
+
output_s3_uri: Optional S3 URI for output data config
|
| 174 |
+
|
| 175 |
+
Returns:
|
| 176 |
+
TwelveLabsAsyncInvokeRequest: The wrapped async invoke request
|
| 177 |
+
"""
|
| 178 |
+
import urllib.parse
|
| 179 |
+
|
| 180 |
+
# Clean the model ID
|
| 181 |
+
unquoted_model_id = urllib.parse.unquote(model_id)
|
| 182 |
+
if unquoted_model_id.startswith("async_invoke/"):
|
| 183 |
+
unquoted_model_id = unquoted_model_id.replace("async_invoke/", "")
|
| 184 |
+
|
| 185 |
+
# Validate that the S3 URI is not empty
|
| 186 |
+
if not output_s3_uri or output_s3_uri.strip() == "":
|
| 187 |
+
raise ValueError("output_s3_uri cannot be empty for async invoke requests")
|
| 188 |
+
|
| 189 |
+
return TwelveLabsAsyncInvokeRequest(
|
| 190 |
+
modelId=unquoted_model_id,
|
| 191 |
+
modelInput=model_input,
|
| 192 |
+
outputDataConfig=TwelveLabsOutputDataConfig(
|
| 193 |
+
s3OutputDataConfig=TwelveLabsS3OutputDataConfig(s3Uri=output_s3_uri)
|
| 194 |
+
),
|
| 195 |
+
)
|
| 196 |
+
|
| 197 |
+
def _transform_response(
|
| 198 |
+
self, response_list: List[dict], model: str
|
| 199 |
+
) -> EmbeddingResponse:
|
| 200 |
+
"""
|
| 201 |
+
Transform TwelveLabs response to OpenAI format.
|
| 202 |
+
Handles the actual TwelveLabs response format: {"data": [{"embedding": [...]}]}
|
| 203 |
+
"""
|
| 204 |
+
embeddings: List[Embedding] = []
|
| 205 |
+
total_tokens = 0
|
| 206 |
+
|
| 207 |
+
for response in response_list:
|
| 208 |
+
# TwelveLabs response format has a "data" field containing the embeddings
|
| 209 |
+
if "data" in response and isinstance(response["data"], list):
|
| 210 |
+
for item in response["data"]:
|
| 211 |
+
if "embedding" in item:
|
| 212 |
+
# Single embedding response
|
| 213 |
+
embedding = Embedding(
|
| 214 |
+
embedding=item["embedding"],
|
| 215 |
+
index=len(embeddings),
|
| 216 |
+
object="embedding",
|
| 217 |
+
)
|
| 218 |
+
embeddings.append(embedding)
|
| 219 |
+
|
| 220 |
+
# Estimate token count (rough approximation)
|
| 221 |
+
if "inputTextTokenCount" in item:
|
| 222 |
+
total_tokens += item["inputTextTokenCount"]
|
| 223 |
+
else:
|
| 224 |
+
# Rough estimate: 1 token per 4 characters for text, or use embedding size
|
| 225 |
+
total_tokens += len(item["embedding"]) // 4
|
| 226 |
+
elif "embedding" in response:
|
| 227 |
+
# Direct embedding response (fallback for other formats)
|
| 228 |
+
embedding = Embedding(
|
| 229 |
+
embedding=response["embedding"],
|
| 230 |
+
index=len(embeddings),
|
| 231 |
+
object="embedding",
|
| 232 |
+
)
|
| 233 |
+
embeddings.append(embedding)
|
| 234 |
+
|
| 235 |
+
# Estimate token count (rough approximation)
|
| 236 |
+
if "inputTextTokenCount" in response:
|
| 237 |
+
total_tokens += response["inputTextTokenCount"]
|
| 238 |
+
else:
|
| 239 |
+
# Rough estimate: 1 token per 4 characters for text
|
| 240 |
+
total_tokens += len(response.get("inputText", "")) // 4
|
| 241 |
+
elif "embeddings" in response:
|
| 242 |
+
# Multiple embeddings response (from video/audio)
|
| 243 |
+
for i, emb in enumerate(response["embeddings"]):
|
| 244 |
+
embedding = Embedding(
|
| 245 |
+
embedding=emb["embedding"],
|
| 246 |
+
index=len(embeddings),
|
| 247 |
+
object="embedding",
|
| 248 |
+
)
|
| 249 |
+
embeddings.append(embedding)
|
| 250 |
+
total_tokens += len(emb["embedding"]) // 4 # Rough estimate
|
| 251 |
+
|
| 252 |
+
usage = Usage(prompt_tokens=total_tokens, total_tokens=total_tokens)
|
| 253 |
+
|
| 254 |
+
return EmbeddingResponse(data=embeddings, model=model, usage=usage)
|
| 255 |
+
|
| 256 |
+
def _transform_async_invoke_response(
|
| 257 |
+
self, response: dict, model: str
|
| 258 |
+
) -> EmbeddingResponse:
|
| 259 |
+
"""
|
| 260 |
+
Transform async invoke response (invocation ARN) to OpenAI format.
|
| 261 |
+
|
| 262 |
+
AWS async invoke returns:
|
| 263 |
+
{
|
| 264 |
+
"invocationArn": "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
We transform this to a job-like embedding response:
|
| 268 |
+
{
|
| 269 |
+
"object": "list",
|
| 270 |
+
"data": [
|
| 271 |
+
{
|
| 272 |
+
"object": "embedding_job_id:1234567890",
|
| 273 |
+
"embedding": [],
|
| 274 |
+
"index": 0
|
| 275 |
+
}
|
| 276 |
+
],
|
| 277 |
+
"model": "model",
|
| 278 |
+
"usage": {}
|
| 279 |
+
}
|
| 280 |
+
"""
|
| 281 |
+
invocation_arn = response.get("invocationArn", "")
|
| 282 |
+
|
| 283 |
+
# Create a placeholder embedding object for the job
|
| 284 |
+
embedding = Embedding(
|
| 285 |
+
embedding=[], # Empty embedding for async jobs
|
| 286 |
+
index=0,
|
| 287 |
+
object="embedding",
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
# Create usage object (empty for async jobs)
|
| 291 |
+
usage = Usage(prompt_tokens=0, total_tokens=0)
|
| 292 |
+
|
| 293 |
+
# Create hidden params with job ID
|
| 294 |
+
from litellm.types.llms.base import HiddenParams
|
| 295 |
+
|
| 296 |
+
hidden_params = HiddenParams()
|
| 297 |
+
setattr(hidden_params, "_invocation_arn", invocation_arn)
|
| 298 |
+
|
| 299 |
+
return EmbeddingResponse(
|
| 300 |
+
data=[embedding],
|
| 301 |
+
model=model,
|
| 302 |
+
usage=usage,
|
| 303 |
+
hidden_params=hidden_params,
|
| 304 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/files/handler.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import base64
|
| 3 |
+
from typing import Any, Coroutine, Optional, Tuple, Union
|
| 4 |
+
|
| 5 |
+
import httpx
|
| 6 |
+
|
| 7 |
+
from litellm import LlmProviders
|
| 8 |
+
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
| 9 |
+
from litellm.types.llms.openai import (
|
| 10 |
+
FileContentRequest,
|
| 11 |
+
HttpxBinaryResponseContent,
|
| 12 |
+
)
|
| 13 |
+
from litellm.types.utils import SpecialEnums
|
| 14 |
+
|
| 15 |
+
from ..base_aws_llm import BaseAWSLLM
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class BedrockFilesHandler(BaseAWSLLM):
|
| 19 |
+
"""
|
| 20 |
+
Handles downloading files from S3 for Bedrock batch processing.
|
| 21 |
+
|
| 22 |
+
This implementation downloads files from S3 buckets where Bedrock
|
| 23 |
+
stores batch output files.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
def __init__(self):
|
| 27 |
+
super().__init__()
|
| 28 |
+
self.async_httpx_client = get_async_httpx_client(
|
| 29 |
+
llm_provider=LlmProviders.BEDROCK,
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
def _extract_s3_uri_from_file_id(self, file_id: str) -> str:
|
| 33 |
+
"""
|
| 34 |
+
Extract S3 URI from encoded file ID.
|
| 35 |
+
|
| 36 |
+
The file ID can be in two formats:
|
| 37 |
+
1. Base64-encoded unified file ID containing: llm_output_file_id,s3://bucket/path
|
| 38 |
+
2. Direct S3 URI: s3://bucket/path
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
file_id: Encoded file ID or direct S3 URI
|
| 42 |
+
|
| 43 |
+
Returns:
|
| 44 |
+
S3 URI (e.g., "s3://bucket-name/path/to/file")
|
| 45 |
+
"""
|
| 46 |
+
# First, try to decode if it's a base64-encoded unified file ID
|
| 47 |
+
try:
|
| 48 |
+
# Add padding if needed
|
| 49 |
+
padded = file_id + "=" * (-len(file_id) % 4)
|
| 50 |
+
decoded = base64.urlsafe_b64decode(padded).decode()
|
| 51 |
+
|
| 52 |
+
# Check if it's a unified file ID format
|
| 53 |
+
if decoded.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value):
|
| 54 |
+
# Extract llm_output_file_id from the decoded string
|
| 55 |
+
if "llm_output_file_id," in decoded:
|
| 56 |
+
s3_uri = decoded.split("llm_output_file_id,")[1].split(";")[0]
|
| 57 |
+
return s3_uri
|
| 58 |
+
except Exception:
|
| 59 |
+
pass
|
| 60 |
+
|
| 61 |
+
# If not base64 encoded or doesn't contain llm_output_file_id, assume it's already an S3 URI
|
| 62 |
+
if file_id.startswith("s3://"):
|
| 63 |
+
return file_id
|
| 64 |
+
|
| 65 |
+
# If it doesn't start with s3://, assume it's a direct S3 URI and add the prefix
|
| 66 |
+
return f"s3://{file_id}"
|
| 67 |
+
|
| 68 |
+
def _parse_s3_uri(self, s3_uri: str) -> Tuple[str, str]:
|
| 69 |
+
"""
|
| 70 |
+
Parse S3 URI to extract bucket name and object key.
|
| 71 |
+
|
| 72 |
+
Args:
|
| 73 |
+
s3_uri: S3 URI (e.g., "s3://bucket-name/path/to/file")
|
| 74 |
+
|
| 75 |
+
Returns:
|
| 76 |
+
Tuple of (bucket_name, object_key)
|
| 77 |
+
"""
|
| 78 |
+
if not s3_uri.startswith("s3://"):
|
| 79 |
+
raise ValueError(
|
| 80 |
+
f"Invalid S3 URI format: {s3_uri}. Expected format: s3://bucket-name/path/to/file"
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
# Remove 's3://' prefix
|
| 84 |
+
path = s3_uri[5:]
|
| 85 |
+
|
| 86 |
+
if "/" in path:
|
| 87 |
+
bucket_name, object_key = path.split("/", 1)
|
| 88 |
+
else:
|
| 89 |
+
bucket_name = path
|
| 90 |
+
object_key = ""
|
| 91 |
+
|
| 92 |
+
return bucket_name, object_key
|
| 93 |
+
|
| 94 |
+
async def afile_content(
|
| 95 |
+
self,
|
| 96 |
+
file_content_request: FileContentRequest,
|
| 97 |
+
optional_params: dict,
|
| 98 |
+
timeout: Union[float, httpx.Timeout],
|
| 99 |
+
max_retries: Optional[int],
|
| 100 |
+
) -> HttpxBinaryResponseContent:
|
| 101 |
+
"""
|
| 102 |
+
Download file content from S3 bucket for Bedrock files.
|
| 103 |
+
|
| 104 |
+
Args:
|
| 105 |
+
file_content_request: Contains file_id (encoded or S3 URI)
|
| 106 |
+
optional_params: Optional parameters containing AWS credentials
|
| 107 |
+
timeout: Request timeout
|
| 108 |
+
max_retries: Max retry attempts
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
HttpxBinaryResponseContent: Binary content wrapped in compatible response format
|
| 112 |
+
"""
|
| 113 |
+
import boto3
|
| 114 |
+
from botocore.credentials import Credentials
|
| 115 |
+
|
| 116 |
+
file_id = file_content_request.get("file_id")
|
| 117 |
+
if not file_id:
|
| 118 |
+
raise ValueError("file_id is required in file_content_request")
|
| 119 |
+
|
| 120 |
+
# Extract S3 URI from file ID
|
| 121 |
+
s3_uri = self._extract_s3_uri_from_file_id(file_id)
|
| 122 |
+
bucket_name, object_key = self._parse_s3_uri(s3_uri)
|
| 123 |
+
|
| 124 |
+
# Get AWS credentials
|
| 125 |
+
aws_region_name = self._get_aws_region_name(
|
| 126 |
+
optional_params=optional_params, model=""
|
| 127 |
+
)
|
| 128 |
+
credentials: Credentials = self.get_credentials(
|
| 129 |
+
aws_access_key_id=optional_params.get("aws_access_key_id"),
|
| 130 |
+
aws_secret_access_key=optional_params.get("aws_secret_access_key"),
|
| 131 |
+
aws_session_token=optional_params.get("aws_session_token"),
|
| 132 |
+
aws_region_name=aws_region_name,
|
| 133 |
+
aws_session_name=optional_params.get("aws_session_name"),
|
| 134 |
+
aws_profile_name=optional_params.get("aws_profile_name"),
|
| 135 |
+
aws_role_name=optional_params.get("aws_role_name"),
|
| 136 |
+
aws_web_identity_token=optional_params.get("aws_web_identity_token"),
|
| 137 |
+
aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
# Create S3 client
|
| 141 |
+
s3_client = boto3.client(
|
| 142 |
+
"s3",
|
| 143 |
+
aws_access_key_id=credentials.access_key,
|
| 144 |
+
aws_secret_access_key=credentials.secret_key,
|
| 145 |
+
aws_session_token=credentials.token,
|
| 146 |
+
region_name=aws_region_name,
|
| 147 |
+
verify=self._get_ssl_verify(),
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
# Download file from S3
|
| 151 |
+
try:
|
| 152 |
+
response = s3_client.get_object(Bucket=bucket_name, Key=object_key)
|
| 153 |
+
file_content = response["Body"].read()
|
| 154 |
+
except Exception as e:
|
| 155 |
+
raise ValueError(
|
| 156 |
+
f"Failed to download file from S3: {s3_uri}. Error: {str(e)}"
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
# Create mock HTTP response
|
| 160 |
+
mock_response = httpx.Response(
|
| 161 |
+
status_code=200,
|
| 162 |
+
content=file_content,
|
| 163 |
+
headers={"content-type": "application/octet-stream"},
|
| 164 |
+
request=httpx.Request(method="GET", url=s3_uri),
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
return HttpxBinaryResponseContent(response=mock_response)
|
| 168 |
+
|
| 169 |
+
def file_content(
|
| 170 |
+
self,
|
| 171 |
+
_is_async: bool,
|
| 172 |
+
file_content_request: FileContentRequest,
|
| 173 |
+
api_base: Optional[str],
|
| 174 |
+
optional_params: dict,
|
| 175 |
+
timeout: Union[float, httpx.Timeout],
|
| 176 |
+
max_retries: Optional[int],
|
| 177 |
+
) -> Union[
|
| 178 |
+
HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
|
| 179 |
+
]:
|
| 180 |
+
"""
|
| 181 |
+
Download file content from S3 bucket for Bedrock files.
|
| 182 |
+
Supports both sync and async operations.
|
| 183 |
+
|
| 184 |
+
Args:
|
| 185 |
+
_is_async: Whether to run asynchronously
|
| 186 |
+
file_content_request: Contains file_id (encoded or S3 URI)
|
| 187 |
+
api_base: API base (unused for S3 operations)
|
| 188 |
+
optional_params: Optional parameters containing AWS credentials
|
| 189 |
+
timeout: Request timeout
|
| 190 |
+
max_retries: Max retry attempts
|
| 191 |
+
|
| 192 |
+
Returns:
|
| 193 |
+
HttpxBinaryResponseContent or Coroutine: Binary content wrapped in compatible response format
|
| 194 |
+
"""
|
| 195 |
+
if _is_async:
|
| 196 |
+
return self.afile_content(
|
| 197 |
+
file_content_request=file_content_request,
|
| 198 |
+
optional_params=optional_params,
|
| 199 |
+
timeout=timeout,
|
| 200 |
+
max_retries=max_retries,
|
| 201 |
+
)
|
| 202 |
+
else:
|
| 203 |
+
return asyncio.run(
|
| 204 |
+
self.afile_content(
|
| 205 |
+
file_content_request=file_content_request,
|
| 206 |
+
optional_params=optional_params,
|
| 207 |
+
timeout=timeout,
|
| 208 |
+
max_retries=max_retries,
|
| 209 |
+
)
|
| 210 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/files/transformation.py
ADDED
|
@@ -0,0 +1,786 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
import time
|
| 4 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 5 |
+
|
| 6 |
+
import httpx
|
| 7 |
+
from httpx import Headers, Response
|
| 8 |
+
from openai.types.file_deleted import FileDeleted
|
| 9 |
+
|
| 10 |
+
from litellm._logging import verbose_logger
|
| 11 |
+
from litellm._uuid import uuid
|
| 12 |
+
from litellm.files.utils import FilesAPIUtils
|
| 13 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
| 14 |
+
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
| 15 |
+
from litellm.llms.base_llm.files.transformation import (
|
| 16 |
+
BaseFilesConfig,
|
| 17 |
+
LiteLLMLoggingObj,
|
| 18 |
+
)
|
| 19 |
+
from litellm.types.llms.openai import (
|
| 20 |
+
AllMessageValues,
|
| 21 |
+
CreateFileRequest,
|
| 22 |
+
FileTypes,
|
| 23 |
+
HttpxBinaryResponseContent,
|
| 24 |
+
OpenAICreateFileRequestOptionalParams,
|
| 25 |
+
OpenAIFileObject,
|
| 26 |
+
PathLike,
|
| 27 |
+
)
|
| 28 |
+
from litellm.types.utils import ExtractedFileData, LlmProviders
|
| 29 |
+
from litellm.utils import get_llm_provider
|
| 30 |
+
|
| 31 |
+
from ..base_aws_llm import BaseAWSLLM
|
| 32 |
+
from ..common_utils import BedrockError
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
| 36 |
+
"""
|
| 37 |
+
Config for Bedrock Files - handles S3 uploads for Bedrock batch processing
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
def __init__(self):
|
| 41 |
+
self.jsonl_transformation = BedrockJsonlFilesTransformation()
|
| 42 |
+
super().__init__()
|
| 43 |
+
|
| 44 |
+
@property
|
| 45 |
+
def custom_llm_provider(self) -> LlmProviders:
|
| 46 |
+
return LlmProviders.BEDROCK
|
| 47 |
+
|
| 48 |
+
@property
|
| 49 |
+
def file_upload_http_method(self) -> str:
|
| 50 |
+
"""
|
| 51 |
+
Bedrock files are uploaded to S3, which requires PUT requests
|
| 52 |
+
"""
|
| 53 |
+
return "PUT"
|
| 54 |
+
|
| 55 |
+
def validate_environment(
|
| 56 |
+
self,
|
| 57 |
+
headers: dict,
|
| 58 |
+
model: str,
|
| 59 |
+
messages: List[AllMessageValues],
|
| 60 |
+
optional_params: dict,
|
| 61 |
+
litellm_params: dict,
|
| 62 |
+
api_key: Optional[str] = None,
|
| 63 |
+
api_base: Optional[str] = None,
|
| 64 |
+
) -> dict:
|
| 65 |
+
# No additional headers needed for S3 uploads - AWS credentials handled by BaseAWSLLM
|
| 66 |
+
return headers
|
| 67 |
+
|
| 68 |
+
def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
|
| 69 |
+
"""
|
| 70 |
+
Helper to extract content from various OpenAI file types and return as string.
|
| 71 |
+
|
| 72 |
+
Handles:
|
| 73 |
+
- Direct content (str, bytes, IO[bytes])
|
| 74 |
+
- Tuple formats: (filename, content, [content_type], [headers])
|
| 75 |
+
- PathLike objects
|
| 76 |
+
"""
|
| 77 |
+
content: Union[str, bytes] = b""
|
| 78 |
+
# Extract file content from tuple if necessary
|
| 79 |
+
if isinstance(openai_file_content, tuple):
|
| 80 |
+
# Take the second element which is always the file content
|
| 81 |
+
file_content = openai_file_content[1]
|
| 82 |
+
else:
|
| 83 |
+
file_content = openai_file_content
|
| 84 |
+
|
| 85 |
+
# Handle different file content types
|
| 86 |
+
if isinstance(file_content, str):
|
| 87 |
+
# String content can be used directly
|
| 88 |
+
content = file_content
|
| 89 |
+
elif isinstance(file_content, bytes):
|
| 90 |
+
# Bytes content can be decoded
|
| 91 |
+
content = file_content
|
| 92 |
+
elif isinstance(file_content, PathLike): # PathLike
|
| 93 |
+
with open(str(file_content), "rb") as f:
|
| 94 |
+
content = f.read()
|
| 95 |
+
elif hasattr(file_content, "read"): # IO[bytes]
|
| 96 |
+
# File-like objects need to be read
|
| 97 |
+
content = file_content.read()
|
| 98 |
+
|
| 99 |
+
# Ensure content is string
|
| 100 |
+
if isinstance(content, bytes):
|
| 101 |
+
content = content.decode("utf-8")
|
| 102 |
+
|
| 103 |
+
return content
|
| 104 |
+
|
| 105 |
+
def _get_s3_object_name_from_batch_jsonl(
|
| 106 |
+
self,
|
| 107 |
+
openai_jsonl_content: List[Dict[str, Any]],
|
| 108 |
+
) -> str:
|
| 109 |
+
"""
|
| 110 |
+
Gets a unique S3 object name for the Bedrock batch processing job
|
| 111 |
+
|
| 112 |
+
named as: litellm-bedrock-files/{model}/{uuid}
|
| 113 |
+
"""
|
| 114 |
+
_model = openai_jsonl_content[0].get("body", {}).get("model", "")
|
| 115 |
+
# Remove bedrock/ prefix if present
|
| 116 |
+
if _model.startswith("bedrock/"):
|
| 117 |
+
_model = _model[8:]
|
| 118 |
+
|
| 119 |
+
# Replace colons with hyphens for Bedrock S3 URI compliance
|
| 120 |
+
_model = _model.replace(":", "-")
|
| 121 |
+
|
| 122 |
+
object_name = f"litellm-bedrock-files-{_model}-{uuid.uuid4()}.jsonl"
|
| 123 |
+
return object_name
|
| 124 |
+
|
| 125 |
+
def get_object_name(
|
| 126 |
+
self, extracted_file_data: ExtractedFileData, purpose: str
|
| 127 |
+
) -> str:
|
| 128 |
+
"""
|
| 129 |
+
Get the object name for the request
|
| 130 |
+
"""
|
| 131 |
+
extracted_file_data_content = extracted_file_data.get("content")
|
| 132 |
+
|
| 133 |
+
if extracted_file_data_content is None:
|
| 134 |
+
raise ValueError("file content is required")
|
| 135 |
+
|
| 136 |
+
if purpose == "batch":
|
| 137 |
+
## 1. If jsonl, check if there's a model name
|
| 138 |
+
file_content = self._get_content_from_openai_file(
|
| 139 |
+
extracted_file_data_content
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
# Split into lines and parse each line as JSON
|
| 143 |
+
openai_jsonl_content = [
|
| 144 |
+
json.loads(line) for line in file_content.splitlines() if line.strip()
|
| 145 |
+
]
|
| 146 |
+
if len(openai_jsonl_content) > 0:
|
| 147 |
+
return self._get_s3_object_name_from_batch_jsonl(openai_jsonl_content)
|
| 148 |
+
|
| 149 |
+
## 2. If not jsonl, return the filename
|
| 150 |
+
filename = extracted_file_data.get("filename")
|
| 151 |
+
if filename:
|
| 152 |
+
return filename
|
| 153 |
+
## 3. If no file name, return timestamp
|
| 154 |
+
return str(int(time.time()))
|
| 155 |
+
|
| 156 |
+
def get_complete_file_url(
|
| 157 |
+
self,
|
| 158 |
+
api_base: Optional[str],
|
| 159 |
+
api_key: Optional[str],
|
| 160 |
+
model: str,
|
| 161 |
+
optional_params: Dict,
|
| 162 |
+
litellm_params: Dict,
|
| 163 |
+
data: CreateFileRequest,
|
| 164 |
+
) -> str:
|
| 165 |
+
"""
|
| 166 |
+
Get the complete S3 URL for the file upload request
|
| 167 |
+
"""
|
| 168 |
+
bucket_name = litellm_params.get("s3_bucket_name") or os.getenv(
|
| 169 |
+
"AWS_S3_BUCKET_NAME"
|
| 170 |
+
)
|
| 171 |
+
if not bucket_name:
|
| 172 |
+
raise ValueError(
|
| 173 |
+
"S3 bucket_name is required. Set 's3_bucket_name' in litellm_params or AWS_S3_BUCKET_NAME env var"
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
s3_region_name = litellm_params.get("s3_region_name") or optional_params.get(
|
| 177 |
+
"s3_region_name"
|
| 178 |
+
)
|
| 179 |
+
aws_region_name = s3_region_name or self._get_aws_region_name(
|
| 180 |
+
optional_params, model
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
file_data = data.get("file")
|
| 184 |
+
purpose = data.get("purpose")
|
| 185 |
+
if file_data is None:
|
| 186 |
+
raise ValueError("file is required")
|
| 187 |
+
if purpose is None:
|
| 188 |
+
raise ValueError("purpose is required")
|
| 189 |
+
extracted_file_data = extract_file_data(file_data)
|
| 190 |
+
object_name = self.get_object_name(extracted_file_data, purpose)
|
| 191 |
+
|
| 192 |
+
# S3 endpoint URL format
|
| 193 |
+
s3_endpoint_url = (
|
| 194 |
+
optional_params.get("s3_endpoint_url")
|
| 195 |
+
or f"https://s3.{aws_region_name}.amazonaws.com"
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
return f"{s3_endpoint_url}/{bucket_name}/{object_name}"
|
| 199 |
+
|
| 200 |
+
def get_supported_openai_params(
|
| 201 |
+
self, model: str
|
| 202 |
+
) -> List[OpenAICreateFileRequestOptionalParams]:
|
| 203 |
+
return []
|
| 204 |
+
|
| 205 |
+
def map_openai_params(
|
| 206 |
+
self,
|
| 207 |
+
non_default_params: dict,
|
| 208 |
+
optional_params: dict,
|
| 209 |
+
model: str,
|
| 210 |
+
drop_params: bool,
|
| 211 |
+
) -> dict:
|
| 212 |
+
return optional_params
|
| 213 |
+
|
| 214 |
+
# Providers whose InvokeModel body uses the Converse API format
|
| 215 |
+
# (messages + inferenceConfig + image blocks). Nova is the primary
|
| 216 |
+
# example; add others here as they adopt the same schema.
|
| 217 |
+
CONVERSE_INVOKE_PROVIDERS = ("nova",)
|
| 218 |
+
|
| 219 |
+
def _map_openai_to_bedrock_params(
|
| 220 |
+
self,
|
| 221 |
+
openai_request_body: Dict[str, Any],
|
| 222 |
+
provider: Optional[str] = None,
|
| 223 |
+
) -> Dict[str, Any]:
|
| 224 |
+
"""
|
| 225 |
+
Transform OpenAI request body to Bedrock-compatible modelInput
|
| 226 |
+
parameters using existing transformation logic.
|
| 227 |
+
|
| 228 |
+
Routes to the correct per-provider transformation so that the
|
| 229 |
+
resulting dict matches the InvokeModel body that Bedrock expects
|
| 230 |
+
for batch inference.
|
| 231 |
+
"""
|
| 232 |
+
from litellm.types.utils import LlmProviders
|
| 233 |
+
|
| 234 |
+
_model = openai_request_body.get("model", "")
|
| 235 |
+
messages = openai_request_body.get("messages", [])
|
| 236 |
+
optional_params = {
|
| 237 |
+
k: v
|
| 238 |
+
for k, v in openai_request_body.items()
|
| 239 |
+
if k not in ["model", "messages"]
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
# --- Anthropic: use existing AmazonAnthropicClaudeConfig ---
|
| 243 |
+
if provider == LlmProviders.ANTHROPIC:
|
| 244 |
+
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
| 245 |
+
AmazonAnthropicClaudeConfig,
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
config = AmazonAnthropicClaudeConfig()
|
| 249 |
+
mapped_params = config.map_openai_params(
|
| 250 |
+
non_default_params={},
|
| 251 |
+
optional_params=optional_params,
|
| 252 |
+
model=_model,
|
| 253 |
+
drop_params=False,
|
| 254 |
+
)
|
| 255 |
+
return config.transform_request(
|
| 256 |
+
model=_model,
|
| 257 |
+
messages=messages,
|
| 258 |
+
optional_params=mapped_params,
|
| 259 |
+
litellm_params={},
|
| 260 |
+
headers={},
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
# --- Converse API providers (e.g. Nova): use AmazonConverseConfig
|
| 264 |
+
# to correctly convert image_url blocks to Bedrock image format
|
| 265 |
+
# and wrap inference params inside inferenceConfig. ---
|
| 266 |
+
if provider in self.CONVERSE_INVOKE_PROVIDERS:
|
| 267 |
+
from litellm.llms.bedrock.chat.converse_transformation import (
|
| 268 |
+
AmazonConverseConfig,
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
converse_config = AmazonConverseConfig()
|
| 272 |
+
mapped_params = converse_config.map_openai_params(
|
| 273 |
+
non_default_params=optional_params,
|
| 274 |
+
optional_params={},
|
| 275 |
+
model=_model,
|
| 276 |
+
drop_params=False,
|
| 277 |
+
)
|
| 278 |
+
return converse_config.transform_request(
|
| 279 |
+
model=_model,
|
| 280 |
+
messages=messages,
|
| 281 |
+
optional_params=mapped_params,
|
| 282 |
+
litellm_params={},
|
| 283 |
+
headers={},
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
# --- All other providers: passthrough (OpenAI-compatible models
|
| 287 |
+
# like openai.gpt-oss-*, qwen, deepseek, etc.) ---
|
| 288 |
+
return {
|
| 289 |
+
"messages": messages,
|
| 290 |
+
**optional_params,
|
| 291 |
+
}
|
| 292 |
+
|
| 293 |
+
def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
| 294 |
+
self, openai_jsonl_content: List[Dict[str, Any]]
|
| 295 |
+
) -> List[Dict[str, Any]]:
|
| 296 |
+
"""
|
| 297 |
+
Transforms OpenAI JSONL content to Bedrock batch format
|
| 298 |
+
|
| 299 |
+
Bedrock batch format: { "recordId": "alphanumeric string", "modelInput": {JSON body} }
|
| 300 |
+
Example:
|
| 301 |
+
{
|
| 302 |
+
"recordId": "CALL0000001",
|
| 303 |
+
"modelInput": {
|
| 304 |
+
"anthropic_version": "bedrock-2023-05-31",
|
| 305 |
+
"max_tokens": 1024,
|
| 306 |
+
"messages": [
|
| 307 |
+
{
|
| 308 |
+
"role": "user",
|
| 309 |
+
"content": [{"type": "text", "text": "Hello"}]
|
| 310 |
+
}
|
| 311 |
+
]
|
| 312 |
+
}
|
| 313 |
+
}
|
| 314 |
+
"""
|
| 315 |
+
|
| 316 |
+
bedrock_jsonl_content = []
|
| 317 |
+
for idx, _openai_jsonl_content in enumerate(openai_jsonl_content):
|
| 318 |
+
# Extract the request body from OpenAI format
|
| 319 |
+
openai_body = _openai_jsonl_content.get("body", {})
|
| 320 |
+
model = openai_body.get("model", "")
|
| 321 |
+
|
| 322 |
+
try:
|
| 323 |
+
model, _, _, _ = get_llm_provider(
|
| 324 |
+
model=model,
|
| 325 |
+
custom_llm_provider=None,
|
| 326 |
+
)
|
| 327 |
+
except Exception as e:
|
| 328 |
+
verbose_logger.exception(
|
| 329 |
+
f"litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - {str(e)}"
|
| 330 |
+
)
|
| 331 |
+
|
| 332 |
+
# Determine provider from model name
|
| 333 |
+
provider = self.get_bedrock_invoke_provider(model)
|
| 334 |
+
|
| 335 |
+
# Transform to Bedrock modelInput format
|
| 336 |
+
model_input = self._map_openai_to_bedrock_params(
|
| 337 |
+
openai_request_body=openai_body, provider=provider
|
| 338 |
+
)
|
| 339 |
+
|
| 340 |
+
# Create Bedrock batch record
|
| 341 |
+
record_id = _openai_jsonl_content.get(
|
| 342 |
+
"custom_id", f"CALL{str(idx).zfill(7)}"
|
| 343 |
+
)
|
| 344 |
+
bedrock_record = {"recordId": record_id, "modelInput": model_input}
|
| 345 |
+
|
| 346 |
+
bedrock_jsonl_content.append(bedrock_record)
|
| 347 |
+
return bedrock_jsonl_content
|
| 348 |
+
|
| 349 |
+
def transform_create_file_request(
|
| 350 |
+
self,
|
| 351 |
+
model: str,
|
| 352 |
+
create_file_data: CreateFileRequest,
|
| 353 |
+
optional_params: dict,
|
| 354 |
+
litellm_params: dict,
|
| 355 |
+
) -> Union[bytes, str, dict]:
|
| 356 |
+
"""
|
| 357 |
+
Transform file request and return a pre-signed request for S3.
|
| 358 |
+
This keeps the HTTP handler clean by doing all the signing here.
|
| 359 |
+
"""
|
| 360 |
+
file_data = create_file_data.get("file")
|
| 361 |
+
if file_data is None:
|
| 362 |
+
raise ValueError("file is required")
|
| 363 |
+
extracted_file_data = extract_file_data(file_data)
|
| 364 |
+
extracted_file_data_content = extracted_file_data.get("content")
|
| 365 |
+
|
| 366 |
+
if extracted_file_data_content is None:
|
| 367 |
+
raise ValueError("file content is required")
|
| 368 |
+
|
| 369 |
+
# Get and transform the file content
|
| 370 |
+
if FilesAPIUtils.is_batch_jsonl_file(
|
| 371 |
+
create_file_data=create_file_data,
|
| 372 |
+
extracted_file_data=extracted_file_data,
|
| 373 |
+
):
|
| 374 |
+
## Transform JSONL content to Bedrock format
|
| 375 |
+
original_file_content = self._get_content_from_openai_file(
|
| 376 |
+
extracted_file_data_content
|
| 377 |
+
)
|
| 378 |
+
openai_jsonl_content = [
|
| 379 |
+
json.loads(line)
|
| 380 |
+
for line in original_file_content.splitlines()
|
| 381 |
+
if line.strip()
|
| 382 |
+
]
|
| 383 |
+
bedrock_jsonl_content = (
|
| 384 |
+
self._transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
| 385 |
+
openai_jsonl_content
|
| 386 |
+
)
|
| 387 |
+
)
|
| 388 |
+
file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
|
| 389 |
+
elif isinstance(extracted_file_data_content, bytes):
|
| 390 |
+
file_content = extracted_file_data_content.decode("utf-8")
|
| 391 |
+
elif isinstance(extracted_file_data_content, str):
|
| 392 |
+
file_content = extracted_file_data_content
|
| 393 |
+
else:
|
| 394 |
+
raise ValueError("Unsupported file content type")
|
| 395 |
+
|
| 396 |
+
# Get the S3 URL for upload
|
| 397 |
+
api_base = self.get_complete_file_url(
|
| 398 |
+
api_base=None,
|
| 399 |
+
api_key=None,
|
| 400 |
+
model=model,
|
| 401 |
+
optional_params=optional_params,
|
| 402 |
+
litellm_params=litellm_params,
|
| 403 |
+
data=create_file_data,
|
| 404 |
+
)
|
| 405 |
+
|
| 406 |
+
# s3_region_name always wins for S3 operations (same priority as in
|
| 407 |
+
# get_complete_file_url above). Overwrite aws_region_name unconditionally
|
| 408 |
+
# so the SigV4 region matches the URL region, avoiding SignatureDoesNotMatch.
|
| 409 |
+
s3_region_name = litellm_params.get("s3_region_name") or optional_params.get(
|
| 410 |
+
"s3_region_name"
|
| 411 |
+
)
|
| 412 |
+
if s3_region_name:
|
| 413 |
+
optional_params = {**optional_params, "aws_region_name": s3_region_name}
|
| 414 |
+
|
| 415 |
+
# Sign the request and return a pre-signed request object
|
| 416 |
+
signed_headers, signed_body = self._sign_s3_request(
|
| 417 |
+
content=file_content,
|
| 418 |
+
api_base=api_base,
|
| 419 |
+
optional_params=optional_params,
|
| 420 |
+
)
|
| 421 |
+
|
| 422 |
+
litellm_params["upload_url"] = api_base
|
| 423 |
+
|
| 424 |
+
# Return a dict that tells the HTTP handler exactly what to do
|
| 425 |
+
return {
|
| 426 |
+
"method": "PUT",
|
| 427 |
+
"url": api_base,
|
| 428 |
+
"headers": signed_headers,
|
| 429 |
+
"data": signed_body or file_content,
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
def _sign_s3_request(
|
| 433 |
+
self,
|
| 434 |
+
content: str,
|
| 435 |
+
api_base: str,
|
| 436 |
+
optional_params: dict,
|
| 437 |
+
) -> Tuple[dict, str]:
|
| 438 |
+
"""
|
| 439 |
+
Sign S3 PUT request using the same proven logic as S3Logger.
|
| 440 |
+
Reuses the exact pattern from litellm/integrations/s3_v2.py
|
| 441 |
+
"""
|
| 442 |
+
try:
|
| 443 |
+
import hashlib
|
| 444 |
+
|
| 445 |
+
import requests
|
| 446 |
+
from botocore.auth import SigV4Auth
|
| 447 |
+
from botocore.awsrequest import AWSRequest
|
| 448 |
+
except ImportError:
|
| 449 |
+
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
|
| 450 |
+
|
| 451 |
+
# Get AWS credentials using existing methods
|
| 452 |
+
aws_region_name = self._get_aws_region_name(
|
| 453 |
+
optional_params=optional_params, model=""
|
| 454 |
+
)
|
| 455 |
+
credentials = self.get_credentials(
|
| 456 |
+
aws_access_key_id=optional_params.get("aws_access_key_id"),
|
| 457 |
+
aws_secret_access_key=optional_params.get("aws_secret_access_key"),
|
| 458 |
+
aws_session_token=optional_params.get("aws_session_token"),
|
| 459 |
+
aws_region_name=aws_region_name,
|
| 460 |
+
aws_session_name=optional_params.get("aws_session_name"),
|
| 461 |
+
aws_profile_name=optional_params.get("aws_profile_name"),
|
| 462 |
+
aws_role_name=optional_params.get("aws_role_name"),
|
| 463 |
+
aws_web_identity_token=optional_params.get("aws_web_identity_token"),
|
| 464 |
+
aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
# Calculate SHA256 hash of the content (REQUIRED for S3)
|
| 468 |
+
content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
| 469 |
+
|
| 470 |
+
# Prepare headers with required S3 headers (same as s3_v2.py)
|
| 471 |
+
request_headers = {
|
| 472 |
+
"Content-Type": "application/json", # JSONL files are JSON content
|
| 473 |
+
"x-amz-content-sha256": content_hash, # REQUIRED by S3
|
| 474 |
+
"Content-Language": "en",
|
| 475 |
+
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
|
| 476 |
+
}
|
| 477 |
+
|
| 478 |
+
# Use requests.Request to prepare the request (same pattern as s3_v2.py)
|
| 479 |
+
req = requests.Request("PUT", api_base, data=content, headers=request_headers)
|
| 480 |
+
prepped = req.prepare()
|
| 481 |
+
|
| 482 |
+
# Sign the request with S3 service
|
| 483 |
+
aws_request = AWSRequest(
|
| 484 |
+
method=prepped.method,
|
| 485 |
+
url=prepped.url,
|
| 486 |
+
data=prepped.body,
|
| 487 |
+
headers=prepped.headers,
|
| 488 |
+
)
|
| 489 |
+
|
| 490 |
+
# Get region name for non-LLM API calls (same as s3_v2.py)
|
| 491 |
+
signing_region = self.get_aws_region_name_for_non_llm_api_calls(
|
| 492 |
+
aws_region_name=aws_region_name
|
| 493 |
+
)
|
| 494 |
+
|
| 495 |
+
SigV4Auth(credentials, "s3", signing_region).add_auth(aws_request)
|
| 496 |
+
|
| 497 |
+
# Return signed headers and body
|
| 498 |
+
signed_body = aws_request.body
|
| 499 |
+
if isinstance(signed_body, bytes):
|
| 500 |
+
signed_body = signed_body.decode("utf-8")
|
| 501 |
+
elif signed_body is None:
|
| 502 |
+
signed_body = content # Fallback to original content
|
| 503 |
+
|
| 504 |
+
return dict(aws_request.headers), signed_body
|
| 505 |
+
|
| 506 |
+
def _convert_https_url_to_s3_uri(self, https_url: str) -> tuple[str, str]:
|
| 507 |
+
"""
|
| 508 |
+
Convert HTTPS S3 URL to s3:// URI format.
|
| 509 |
+
|
| 510 |
+
Args:
|
| 511 |
+
https_url: HTTPS S3 URL (e.g., "https://s3.us-west-2.amazonaws.com/bucket/key")
|
| 512 |
+
|
| 513 |
+
Returns:
|
| 514 |
+
Tuple of (s3_uri, filename)
|
| 515 |
+
|
| 516 |
+
Example:
|
| 517 |
+
Input: "https://s3.us-west-2.amazonaws.com/litellm-proxy/file.jsonl"
|
| 518 |
+
Output: ("s3://litellm-proxy/file.jsonl", "file.jsonl")
|
| 519 |
+
"""
|
| 520 |
+
import re
|
| 521 |
+
|
| 522 |
+
# Match HTTPS S3 URL patterns
|
| 523 |
+
# Pattern 1: https://s3.region.amazonaws.com/bucket/key
|
| 524 |
+
# Pattern 2: https://bucket.s3.region.amazonaws.com/key
|
| 525 |
+
|
| 526 |
+
pattern1 = r"https://s3\.([^.]+)\.amazonaws\.com/([^/]+)/(.+)"
|
| 527 |
+
pattern2 = r"https://([^.]+)\.s3\.([^.]+)\.amazonaws\.com/(.+)"
|
| 528 |
+
|
| 529 |
+
match1 = re.match(pattern1, https_url)
|
| 530 |
+
match2 = re.match(pattern2, https_url)
|
| 531 |
+
|
| 532 |
+
if match1:
|
| 533 |
+
# Pattern: https://s3.region.amazonaws.com/bucket/key
|
| 534 |
+
region, bucket, key = match1.groups()
|
| 535 |
+
s3_uri = f"s3://{bucket}/{key}"
|
| 536 |
+
elif match2:
|
| 537 |
+
# Pattern: https://bucket.s3.region.amazonaws.com/key
|
| 538 |
+
bucket, region, key = match2.groups()
|
| 539 |
+
s3_uri = f"s3://{bucket}/{key}"
|
| 540 |
+
else:
|
| 541 |
+
# Fallback: try to extract bucket and key from URL path
|
| 542 |
+
from urllib.parse import urlparse
|
| 543 |
+
|
| 544 |
+
parsed = urlparse(https_url)
|
| 545 |
+
path_parts = parsed.path.lstrip("/").split("/", 1)
|
| 546 |
+
if len(path_parts) >= 2:
|
| 547 |
+
bucket, key = path_parts[0], path_parts[1]
|
| 548 |
+
s3_uri = f"s3://{bucket}/{key}"
|
| 549 |
+
else:
|
| 550 |
+
raise ValueError(f"Unable to parse S3 URL: {https_url}")
|
| 551 |
+
|
| 552 |
+
# Extract filename from key
|
| 553 |
+
filename = key.split("/")[-1] if "/" in key else key
|
| 554 |
+
|
| 555 |
+
return s3_uri, filename
|
| 556 |
+
|
| 557 |
+
def transform_create_file_response(
|
| 558 |
+
self,
|
| 559 |
+
model: Optional[str],
|
| 560 |
+
raw_response: Response,
|
| 561 |
+
logging_obj: LiteLLMLoggingObj,
|
| 562 |
+
litellm_params: dict,
|
| 563 |
+
) -> OpenAIFileObject:
|
| 564 |
+
"""
|
| 565 |
+
Transform S3 File upload response into OpenAI-style FileObject
|
| 566 |
+
"""
|
| 567 |
+
# For S3 uploads, we typically get an ETag and other metadata
|
| 568 |
+
response_headers = raw_response.headers
|
| 569 |
+
# Extract S3 object information from the response
|
| 570 |
+
# S3 PUT object returns ETag and other metadata in headers
|
| 571 |
+
content_length = response_headers.get("Content-Length", "0")
|
| 572 |
+
|
| 573 |
+
# Use the actual upload URL that was used for the S3 upload
|
| 574 |
+
upload_url = litellm_params.get("upload_url")
|
| 575 |
+
file_id: str = ""
|
| 576 |
+
filename: str = ""
|
| 577 |
+
if upload_url:
|
| 578 |
+
# Convert HTTPS S3 URL to s3:// URI format
|
| 579 |
+
file_id, filename = self._convert_https_url_to_s3_uri(upload_url)
|
| 580 |
+
|
| 581 |
+
return OpenAIFileObject(
|
| 582 |
+
purpose="batch", # Default purpose for Bedrock files
|
| 583 |
+
id=file_id,
|
| 584 |
+
filename=filename,
|
| 585 |
+
created_at=int(time.time()), # Current timestamp
|
| 586 |
+
status="uploaded",
|
| 587 |
+
bytes=int(content_length) if content_length.isdigit() else 0,
|
| 588 |
+
object="file",
|
| 589 |
+
)
|
| 590 |
+
|
| 591 |
+
def get_error_class(
|
| 592 |
+
self, error_message: str, status_code: int, headers: Union[Dict, Headers]
|
| 593 |
+
) -> BaseLLMException:
|
| 594 |
+
return BedrockError(
|
| 595 |
+
status_code=status_code, message=error_message, headers=headers
|
| 596 |
+
)
|
| 597 |
+
|
| 598 |
+
def transform_retrieve_file_request(
|
| 599 |
+
self,
|
| 600 |
+
file_id: str,
|
| 601 |
+
optional_params: dict,
|
| 602 |
+
litellm_params: dict,
|
| 603 |
+
) -> tuple[str, dict]:
|
| 604 |
+
raise NotImplementedError("BedrockFilesConfig does not support file retrieval")
|
| 605 |
+
|
| 606 |
+
def transform_retrieve_file_response(
|
| 607 |
+
self,
|
| 608 |
+
raw_response: httpx.Response,
|
| 609 |
+
logging_obj: LiteLLMLoggingObj,
|
| 610 |
+
litellm_params: dict,
|
| 611 |
+
) -> OpenAIFileObject:
|
| 612 |
+
raise NotImplementedError("BedrockFilesConfig does not support file retrieval")
|
| 613 |
+
|
| 614 |
+
def transform_delete_file_request(
|
| 615 |
+
self,
|
| 616 |
+
file_id: str,
|
| 617 |
+
optional_params: dict,
|
| 618 |
+
litellm_params: dict,
|
| 619 |
+
) -> tuple[str, dict]:
|
| 620 |
+
raise NotImplementedError("BedrockFilesConfig does not support file deletion")
|
| 621 |
+
|
| 622 |
+
def transform_delete_file_response(
|
| 623 |
+
self,
|
| 624 |
+
raw_response: httpx.Response,
|
| 625 |
+
logging_obj: LiteLLMLoggingObj,
|
| 626 |
+
litellm_params: dict,
|
| 627 |
+
) -> FileDeleted:
|
| 628 |
+
raise NotImplementedError("BedrockFilesConfig does not support file deletion")
|
| 629 |
+
|
| 630 |
+
def transform_list_files_request(
|
| 631 |
+
self,
|
| 632 |
+
purpose: Optional[str],
|
| 633 |
+
optional_params: dict,
|
| 634 |
+
litellm_params: dict,
|
| 635 |
+
) -> tuple[str, dict]:
|
| 636 |
+
raise NotImplementedError("BedrockFilesConfig does not support file listing")
|
| 637 |
+
|
| 638 |
+
def transform_list_files_response(
|
| 639 |
+
self,
|
| 640 |
+
raw_response: httpx.Response,
|
| 641 |
+
logging_obj: LiteLLMLoggingObj,
|
| 642 |
+
litellm_params: dict,
|
| 643 |
+
) -> List[OpenAIFileObject]:
|
| 644 |
+
raise NotImplementedError("BedrockFilesConfig does not support file listing")
|
| 645 |
+
|
| 646 |
+
def transform_file_content_request(
|
| 647 |
+
self,
|
| 648 |
+
file_content_request,
|
| 649 |
+
optional_params: dict,
|
| 650 |
+
litellm_params: dict,
|
| 651 |
+
) -> tuple[str, dict]:
|
| 652 |
+
raise NotImplementedError(
|
| 653 |
+
"BedrockFilesConfig does not support file content retrieval"
|
| 654 |
+
)
|
| 655 |
+
|
| 656 |
+
def transform_file_content_response(
|
| 657 |
+
self,
|
| 658 |
+
raw_response: httpx.Response,
|
| 659 |
+
logging_obj: LiteLLMLoggingObj,
|
| 660 |
+
litellm_params: dict,
|
| 661 |
+
) -> HttpxBinaryResponseContent:
|
| 662 |
+
raise NotImplementedError(
|
| 663 |
+
"BedrockFilesConfig does not support file content retrieval"
|
| 664 |
+
)
|
| 665 |
+
|
| 666 |
+
|
| 667 |
+
class BedrockJsonlFilesTransformation:
|
| 668 |
+
"""
|
| 669 |
+
Transforms OpenAI /v1/files/* requests to Bedrock S3 file uploads for batch processing
|
| 670 |
+
"""
|
| 671 |
+
|
| 672 |
+
def transform_openai_file_content_to_bedrock_file_content(
|
| 673 |
+
self, openai_file_content: Optional[FileTypes] = None
|
| 674 |
+
) -> Tuple[str, str]:
|
| 675 |
+
"""
|
| 676 |
+
Transforms OpenAI FileContentRequest to Bedrock S3 file format
|
| 677 |
+
"""
|
| 678 |
+
|
| 679 |
+
if openai_file_content is None:
|
| 680 |
+
raise ValueError("contents of file are None")
|
| 681 |
+
# Read the content of the file
|
| 682 |
+
file_content = self._get_content_from_openai_file(openai_file_content)
|
| 683 |
+
|
| 684 |
+
# Split into lines and parse each line as JSON
|
| 685 |
+
openai_jsonl_content = [
|
| 686 |
+
json.loads(line) for line in file_content.splitlines() if line.strip()
|
| 687 |
+
]
|
| 688 |
+
bedrock_jsonl_content = (
|
| 689 |
+
self._transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
| 690 |
+
openai_jsonl_content
|
| 691 |
+
)
|
| 692 |
+
)
|
| 693 |
+
bedrock_jsonl_string = "\n".join(
|
| 694 |
+
json.dumps(item) for item in bedrock_jsonl_content
|
| 695 |
+
)
|
| 696 |
+
object_name = self._get_s3_object_name(
|
| 697 |
+
openai_jsonl_content=openai_jsonl_content
|
| 698 |
+
)
|
| 699 |
+
return bedrock_jsonl_string, object_name
|
| 700 |
+
|
| 701 |
+
def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
| 702 |
+
self, openai_jsonl_content: List[Dict[str, Any]]
|
| 703 |
+
):
|
| 704 |
+
"""
|
| 705 |
+
Delegate to the main BedrockFilesConfig transformation method
|
| 706 |
+
"""
|
| 707 |
+
config = BedrockFilesConfig()
|
| 708 |
+
return config._transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
| 709 |
+
openai_jsonl_content
|
| 710 |
+
)
|
| 711 |
+
|
| 712 |
+
def _get_s3_object_name(
|
| 713 |
+
self,
|
| 714 |
+
openai_jsonl_content: List[Dict[str, Any]],
|
| 715 |
+
) -> str:
|
| 716 |
+
"""
|
| 717 |
+
Gets a unique S3 object name for the Bedrock batch processing job
|
| 718 |
+
|
| 719 |
+
named as: litellm-bedrock-files-{model}-{uuid}
|
| 720 |
+
"""
|
| 721 |
+
_model = openai_jsonl_content[0].get("body", {}).get("model", "")
|
| 722 |
+
# Remove bedrock/ prefix if present
|
| 723 |
+
if _model.startswith("bedrock/"):
|
| 724 |
+
_model = _model[8:]
|
| 725 |
+
object_name = f"litellm-bedrock-files-{_model}-{uuid.uuid4()}.jsonl"
|
| 726 |
+
return object_name
|
| 727 |
+
|
| 728 |
+
def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
|
| 729 |
+
"""
|
| 730 |
+
Helper to extract content from various OpenAI file types and return as string.
|
| 731 |
+
|
| 732 |
+
Handles:
|
| 733 |
+
- Direct content (str, bytes, IO[bytes])
|
| 734 |
+
- Tuple formats: (filename, content, [content_type], [headers])
|
| 735 |
+
- PathLike objects
|
| 736 |
+
"""
|
| 737 |
+
content: Union[str, bytes] = b""
|
| 738 |
+
# Extract file content from tuple if necessary
|
| 739 |
+
if isinstance(openai_file_content, tuple):
|
| 740 |
+
# Take the second element which is always the file content
|
| 741 |
+
file_content = openai_file_content[1]
|
| 742 |
+
else:
|
| 743 |
+
file_content = openai_file_content
|
| 744 |
+
|
| 745 |
+
# Handle different file content types
|
| 746 |
+
if isinstance(file_content, str):
|
| 747 |
+
# String content can be used directly
|
| 748 |
+
content = file_content
|
| 749 |
+
elif isinstance(file_content, bytes):
|
| 750 |
+
# Bytes content can be decoded
|
| 751 |
+
content = file_content
|
| 752 |
+
elif isinstance(file_content, PathLike): # PathLike
|
| 753 |
+
with open(str(file_content), "rb") as f:
|
| 754 |
+
content = f.read()
|
| 755 |
+
elif hasattr(file_content, "read"): # IO[bytes]
|
| 756 |
+
# File-like objects need to be read
|
| 757 |
+
content = file_content.read()
|
| 758 |
+
|
| 759 |
+
# Ensure content is string
|
| 760 |
+
if isinstance(content, bytes):
|
| 761 |
+
content = content.decode("utf-8")
|
| 762 |
+
|
| 763 |
+
return content
|
| 764 |
+
|
| 765 |
+
def transform_s3_bucket_response_to_openai_file_object(
|
| 766 |
+
self, create_file_data: CreateFileRequest, s3_upload_response: Dict[str, Any]
|
| 767 |
+
) -> OpenAIFileObject:
|
| 768 |
+
"""
|
| 769 |
+
Transforms S3 Bucket upload file response to OpenAI FileObject
|
| 770 |
+
"""
|
| 771 |
+
# S3 response typically contains ETag, key, etc.
|
| 772 |
+
object_key = s3_upload_response.get("Key", "")
|
| 773 |
+
bucket_name = s3_upload_response.get("Bucket", "")
|
| 774 |
+
|
| 775 |
+
# Extract filename from object key
|
| 776 |
+
filename = object_key.split("/")[-1] if "/" in object_key else object_key
|
| 777 |
+
|
| 778 |
+
return OpenAIFileObject(
|
| 779 |
+
purpose=create_file_data.get("purpose", "batch"),
|
| 780 |
+
id=f"s3://{bucket_name}/{object_key}",
|
| 781 |
+
filename=filename,
|
| 782 |
+
created_at=int(time.time()), # Current timestamp
|
| 783 |
+
status="uploaded",
|
| 784 |
+
bytes=s3_upload_response.get("ContentLength", 0),
|
| 785 |
+
object="file",
|
| 786 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Bedrock Image Edit Module
|
| 3 |
+
|
| 4 |
+
Handles image edit operations for Bedrock stability models.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from .handler import BedrockImageEdit
|
| 8 |
+
|
| 9 |
+
__all__ = ["BedrockImageEdit"]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/handler.py
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Bedrock Image Edit Handler
|
| 3 |
+
|
| 4 |
+
Handles image edit requests for Bedrock stability models.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
| 11 |
+
|
| 12 |
+
import httpx
|
| 13 |
+
from pydantic import BaseModel
|
| 14 |
+
|
| 15 |
+
import litellm
|
| 16 |
+
from litellm._logging import verbose_logger
|
| 17 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
|
| 18 |
+
from litellm.llms.bedrock.image_edit.stability_transformation import (
|
| 19 |
+
BedrockStabilityImageEditConfig,
|
| 20 |
+
)
|
| 21 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 22 |
+
AsyncHTTPHandler,
|
| 23 |
+
HTTPHandler,
|
| 24 |
+
_get_httpx_client,
|
| 25 |
+
get_async_httpx_client,
|
| 26 |
+
)
|
| 27 |
+
from litellm.types.utils import ImageResponse
|
| 28 |
+
|
| 29 |
+
from ..base_aws_llm import BaseAWSLLM
|
| 30 |
+
from ..common_utils import BedrockError
|
| 31 |
+
|
| 32 |
+
if TYPE_CHECKING:
|
| 33 |
+
from botocore.awsrequest import AWSPreparedRequest
|
| 34 |
+
else:
|
| 35 |
+
AWSPreparedRequest = Any
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class BedrockImageEditPreparedRequest(BaseModel):
|
| 39 |
+
"""
|
| 40 |
+
Internal/Helper class for preparing the request for bedrock image edit
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
endpoint_url: str
|
| 44 |
+
prepped: AWSPreparedRequest
|
| 45 |
+
body: bytes
|
| 46 |
+
data: dict
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class BedrockImageEdit(BaseAWSLLM):
|
| 50 |
+
"""
|
| 51 |
+
Bedrock Image Edit handler
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
@classmethod
|
| 55 |
+
def get_config_class(cls, model: str | None):
|
| 56 |
+
if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
|
| 57 |
+
return BedrockStabilityImageEditConfig
|
| 58 |
+
else:
|
| 59 |
+
raise ValueError(f"Unsupported model for bedrock image edit: {model}")
|
| 60 |
+
|
| 61 |
+
def image_edit(
|
| 62 |
+
self,
|
| 63 |
+
model: str,
|
| 64 |
+
image: list,
|
| 65 |
+
prompt: Optional[str],
|
| 66 |
+
model_response: ImageResponse,
|
| 67 |
+
optional_params: dict,
|
| 68 |
+
logging_obj: LitellmLogging,
|
| 69 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 70 |
+
aimage_edit: bool = False,
|
| 71 |
+
api_base: Optional[str] = None,
|
| 72 |
+
extra_headers: Optional[dict] = None,
|
| 73 |
+
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
| 74 |
+
api_key: Optional[str] = None,
|
| 75 |
+
):
|
| 76 |
+
prepared_request = self._prepare_request(
|
| 77 |
+
model=model,
|
| 78 |
+
image=image,
|
| 79 |
+
prompt=prompt,
|
| 80 |
+
optional_params=optional_params,
|
| 81 |
+
api_base=api_base,
|
| 82 |
+
extra_headers=extra_headers,
|
| 83 |
+
logging_obj=logging_obj,
|
| 84 |
+
api_key=api_key,
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
if aimage_edit is True:
|
| 88 |
+
return self.async_image_edit(
|
| 89 |
+
prepared_request=prepared_request,
|
| 90 |
+
timeout=timeout,
|
| 91 |
+
model=model,
|
| 92 |
+
logging_obj=logging_obj,
|
| 93 |
+
prompt=prompt,
|
| 94 |
+
model_response=model_response,
|
| 95 |
+
client=(
|
| 96 |
+
client
|
| 97 |
+
if client is not None and isinstance(client, AsyncHTTPHandler)
|
| 98 |
+
else None
|
| 99 |
+
),
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
if client is None or not isinstance(client, HTTPHandler):
|
| 103 |
+
client = _get_httpx_client()
|
| 104 |
+
try:
|
| 105 |
+
response = client.post(url=prepared_request.endpoint_url, headers=prepared_request.prepped.headers, data=prepared_request.body) # type: ignore
|
| 106 |
+
response.raise_for_status()
|
| 107 |
+
except httpx.HTTPStatusError as err:
|
| 108 |
+
error_code = err.response.status_code
|
| 109 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 110 |
+
except httpx.TimeoutException:
|
| 111 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 112 |
+
|
| 113 |
+
### FORMAT RESPONSE TO OPENAI FORMAT ###
|
| 114 |
+
model_response = self._transform_response_dict_to_openai_response(
|
| 115 |
+
model_response=model_response,
|
| 116 |
+
model=model,
|
| 117 |
+
logging_obj=logging_obj,
|
| 118 |
+
prompt=prompt,
|
| 119 |
+
response=response,
|
| 120 |
+
data=prepared_request.data,
|
| 121 |
+
)
|
| 122 |
+
return model_response
|
| 123 |
+
|
| 124 |
+
async def async_image_edit(
|
| 125 |
+
self,
|
| 126 |
+
prepared_request: BedrockImageEditPreparedRequest,
|
| 127 |
+
timeout: Optional[Union[float, httpx.Timeout]],
|
| 128 |
+
model: str,
|
| 129 |
+
logging_obj: LitellmLogging,
|
| 130 |
+
prompt: Optional[str],
|
| 131 |
+
model_response: ImageResponse,
|
| 132 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 133 |
+
) -> ImageResponse:
|
| 134 |
+
"""
|
| 135 |
+
Asynchronous handler for bedrock image edit
|
| 136 |
+
"""
|
| 137 |
+
async_client = client or get_async_httpx_client(
|
| 138 |
+
llm_provider=litellm.LlmProviders.BEDROCK,
|
| 139 |
+
params={"timeout": timeout},
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
try:
|
| 143 |
+
response = await async_client.post(url=prepared_request.endpoint_url, headers=prepared_request.prepped.headers, data=prepared_request.body) # type: ignore
|
| 144 |
+
response.raise_for_status()
|
| 145 |
+
except httpx.HTTPStatusError as err:
|
| 146 |
+
error_code = err.response.status_code
|
| 147 |
+
raise BedrockError(status_code=error_code, message=err.response.text)
|
| 148 |
+
except httpx.TimeoutException:
|
| 149 |
+
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
| 150 |
+
|
| 151 |
+
### FORMAT RESPONSE TO OPENAI FORMAT ###
|
| 152 |
+
model_response = self._transform_response_dict_to_openai_response(
|
| 153 |
+
model=model,
|
| 154 |
+
logging_obj=logging_obj,
|
| 155 |
+
prompt=prompt,
|
| 156 |
+
response=response,
|
| 157 |
+
data=prepared_request.data,
|
| 158 |
+
model_response=model_response,
|
| 159 |
+
)
|
| 160 |
+
return model_response
|
| 161 |
+
|
| 162 |
+
def _prepare_request(
|
| 163 |
+
self,
|
| 164 |
+
model: str,
|
| 165 |
+
image: list,
|
| 166 |
+
prompt: Optional[str],
|
| 167 |
+
optional_params: dict,
|
| 168 |
+
api_base: Optional[str],
|
| 169 |
+
extra_headers: Optional[dict],
|
| 170 |
+
logging_obj: LitellmLogging,
|
| 171 |
+
api_key: Optional[str],
|
| 172 |
+
) -> BedrockImageEditPreparedRequest:
|
| 173 |
+
"""
|
| 174 |
+
Prepare the request body, headers, and endpoint URL for the Bedrock Image Edit API
|
| 175 |
+
|
| 176 |
+
Args:
|
| 177 |
+
model (str): The model to use for the image edit
|
| 178 |
+
image (list): The images to edit
|
| 179 |
+
prompt (Optional[str]): The prompt for the edit
|
| 180 |
+
optional_params (dict): The optional parameters for the image edit
|
| 181 |
+
api_base (Optional[str]): The base URL for the Bedrock API
|
| 182 |
+
extra_headers (Optional[dict]): The extra headers to include in the request
|
| 183 |
+
logging_obj (LitellmLogging): The logging object to use for logging
|
| 184 |
+
api_key (Optional[str]): The API key to use
|
| 185 |
+
|
| 186 |
+
Returns:
|
| 187 |
+
BedrockImageEditPreparedRequest: The prepared request object
|
| 188 |
+
"""
|
| 189 |
+
boto3_credentials_info = self._get_boto_credentials_from_optional_params(
|
| 190 |
+
optional_params, model
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
# Use the existing ARN-aware provider detection method
|
| 194 |
+
bedrock_provider = self.get_bedrock_invoke_provider(model)
|
| 195 |
+
### SET RUNTIME ENDPOINT ###
|
| 196 |
+
modelId = self.get_bedrock_model_id(
|
| 197 |
+
model=model,
|
| 198 |
+
provider=bedrock_provider,
|
| 199 |
+
optional_params=optional_params,
|
| 200 |
+
)
|
| 201 |
+
_, proxy_endpoint_url = self.get_runtime_endpoint(
|
| 202 |
+
api_base=api_base,
|
| 203 |
+
aws_bedrock_runtime_endpoint=boto3_credentials_info.aws_bedrock_runtime_endpoint,
|
| 204 |
+
aws_region_name=boto3_credentials_info.aws_region_name,
|
| 205 |
+
)
|
| 206 |
+
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
|
| 207 |
+
data = self._get_request_body(
|
| 208 |
+
model=model,
|
| 209 |
+
image=image,
|
| 210 |
+
prompt=prompt,
|
| 211 |
+
optional_params=optional_params,
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
# Make POST Request
|
| 215 |
+
body = json.dumps(data).encode("utf-8")
|
| 216 |
+
headers = {"Content-Type": "application/json"}
|
| 217 |
+
if extra_headers is not None:
|
| 218 |
+
headers = {"Content-Type": "application/json", **extra_headers}
|
| 219 |
+
|
| 220 |
+
prepped = self.get_request_headers(
|
| 221 |
+
credentials=boto3_credentials_info.credentials,
|
| 222 |
+
aws_region_name=boto3_credentials_info.aws_region_name,
|
| 223 |
+
extra_headers=extra_headers,
|
| 224 |
+
endpoint_url=proxy_endpoint_url,
|
| 225 |
+
data=body,
|
| 226 |
+
headers=headers,
|
| 227 |
+
api_key=api_key,
|
| 228 |
+
)
|
| 229 |
+
|
| 230 |
+
## LOGGING
|
| 231 |
+
logging_obj.pre_call(
|
| 232 |
+
input=prompt,
|
| 233 |
+
api_key="",
|
| 234 |
+
additional_args={
|
| 235 |
+
"complete_input_dict": data,
|
| 236 |
+
"api_base": proxy_endpoint_url,
|
| 237 |
+
"headers": prepped.headers,
|
| 238 |
+
},
|
| 239 |
+
)
|
| 240 |
+
return BedrockImageEditPreparedRequest(
|
| 241 |
+
endpoint_url=proxy_endpoint_url,
|
| 242 |
+
prepped=prepped,
|
| 243 |
+
body=body,
|
| 244 |
+
data=data,
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
def _get_request_body(
|
| 248 |
+
self,
|
| 249 |
+
model: str,
|
| 250 |
+
image: list,
|
| 251 |
+
prompt: Optional[str],
|
| 252 |
+
optional_params: dict,
|
| 253 |
+
) -> dict:
|
| 254 |
+
"""
|
| 255 |
+
Get the request body for the Bedrock Image Edit API
|
| 256 |
+
|
| 257 |
+
Checks the model/provider and transforms the request body accordingly
|
| 258 |
+
|
| 259 |
+
Returns:
|
| 260 |
+
dict: The request body to use for the Bedrock Image Edit API
|
| 261 |
+
"""
|
| 262 |
+
config_class = self.get_config_class(model=model)
|
| 263 |
+
config_instance = config_class()
|
| 264 |
+
request_body, _ = config_instance.transform_image_edit_request(
|
| 265 |
+
model=model,
|
| 266 |
+
prompt=prompt,
|
| 267 |
+
image=image[0] if image else None,
|
| 268 |
+
image_edit_optional_request_params=optional_params,
|
| 269 |
+
litellm_params={},
|
| 270 |
+
headers={},
|
| 271 |
+
)
|
| 272 |
+
return dict(request_body)
|
| 273 |
+
|
| 274 |
+
def _transform_response_dict_to_openai_response(
|
| 275 |
+
self,
|
| 276 |
+
model_response: ImageResponse,
|
| 277 |
+
model: str,
|
| 278 |
+
logging_obj: LitellmLogging,
|
| 279 |
+
prompt: Optional[str],
|
| 280 |
+
response: httpx.Response,
|
| 281 |
+
data: dict,
|
| 282 |
+
) -> ImageResponse:
|
| 283 |
+
"""
|
| 284 |
+
Transforms the Image Edit response from Bedrock to OpenAI format
|
| 285 |
+
"""
|
| 286 |
+
|
| 287 |
+
## LOGGING
|
| 288 |
+
if logging_obj is not None:
|
| 289 |
+
logging_obj.post_call(
|
| 290 |
+
input=prompt,
|
| 291 |
+
api_key="",
|
| 292 |
+
original_response=response.text,
|
| 293 |
+
additional_args={"complete_input_dict": data},
|
| 294 |
+
)
|
| 295 |
+
verbose_logger.debug("raw model_response: %s", response.text)
|
| 296 |
+
response_dict = response.json()
|
| 297 |
+
if response_dict is None:
|
| 298 |
+
raise ValueError("Error in response object format, got None")
|
| 299 |
+
|
| 300 |
+
config_class = self.get_config_class(model=model)
|
| 301 |
+
config_instance = config_class()
|
| 302 |
+
|
| 303 |
+
model_response = config_instance.transform_image_edit_response(
|
| 304 |
+
model=model,
|
| 305 |
+
raw_response=response,
|
| 306 |
+
logging_obj=logging_obj,
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
return model_response
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/stability_transformation.py
ADDED
|
@@ -0,0 +1,399 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Bedrock Stability AI Image Edit Transformation
|
| 3 |
+
|
| 4 |
+
Handles transformation between OpenAI-compatible format and Bedrock Stability AI Image Edit API format.
|
| 5 |
+
|
| 6 |
+
Supported models:
|
| 7 |
+
- stability.stable-conservative-upscale-v1:0
|
| 8 |
+
- stability.stable-creative-upscale-v1:0
|
| 9 |
+
- stability.stable-fast-upscale-v1:0
|
| 10 |
+
- stability.stable-outpaint-v1:0
|
| 11 |
+
- stability.stable-image-control-sketch-v1:0
|
| 12 |
+
- stability.stable-image-control-structure-v1:0
|
| 13 |
+
- stability.stable-image-erase-object-v1:0
|
| 14 |
+
- stability.stable-image-inpaint-v1:0
|
| 15 |
+
- stability.stable-image-remove-background-v1:0
|
| 16 |
+
- stability.stable-image-search-recolor-v1:0
|
| 17 |
+
- stability.stable-image-search-replace-v1:0
|
| 18 |
+
- stability.stable-image-style-guide-v1:0
|
| 19 |
+
- stability.stable-style-transfer-v1:0
|
| 20 |
+
|
| 21 |
+
API Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
import base64
|
| 25 |
+
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
|
| 26 |
+
|
| 27 |
+
import httpx
|
| 28 |
+
|
| 29 |
+
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
| 30 |
+
from litellm.types.images.main import ImageEditOptionalRequestParams
|
| 31 |
+
from litellm.types.llms.stability import (
|
| 32 |
+
OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO,
|
| 33 |
+
)
|
| 34 |
+
from litellm.types.router import GenericLiteLLMParams
|
| 35 |
+
from litellm.types.utils import FileTypes, ImageObject, ImageResponse
|
| 36 |
+
from litellm.utils import get_model_info
|
| 37 |
+
|
| 38 |
+
if TYPE_CHECKING:
|
| 39 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 40 |
+
|
| 41 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 42 |
+
else:
|
| 43 |
+
LiteLLMLoggingObj = Any
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class BedrockStabilityImageEditConfig(BaseImageEditConfig):
|
| 47 |
+
"""
|
| 48 |
+
Configuration for Bedrock Stability AI image edit.
|
| 49 |
+
|
| 50 |
+
Supports all Stability image edit operations through Bedrock.
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
@classmethod
|
| 54 |
+
def _is_stability_edit_model(cls, model: Optional[str] = None) -> bool:
|
| 55 |
+
"""
|
| 56 |
+
Returns True if the model is a Bedrock Stability edit model.
|
| 57 |
+
|
| 58 |
+
Bedrock Stability edit models follow this pattern:
|
| 59 |
+
stability.stable-conservative-upscale-v1:0
|
| 60 |
+
stability.stable-creative-upscale-v1:0
|
| 61 |
+
stability.stable-fast-upscale-v1:0
|
| 62 |
+
stability.stable-outpaint-v1:0
|
| 63 |
+
stability.stable-image-inpaint-v1:0
|
| 64 |
+
stability.stable-image-erase-object-v1:0
|
| 65 |
+
etc.
|
| 66 |
+
"""
|
| 67 |
+
if model:
|
| 68 |
+
model_lower = model.lower()
|
| 69 |
+
if "stability." in model_lower and any(
|
| 70 |
+
[
|
| 71 |
+
"upscale" in model_lower,
|
| 72 |
+
"outpaint" in model_lower,
|
| 73 |
+
"inpaint" in model_lower,
|
| 74 |
+
"erase" in model_lower,
|
| 75 |
+
"remove-background" in model_lower,
|
| 76 |
+
"search-recolor" in model_lower,
|
| 77 |
+
"search-replace" in model_lower,
|
| 78 |
+
"control-sketch" in model_lower,
|
| 79 |
+
"control-structure" in model_lower,
|
| 80 |
+
"style-guide" in model_lower,
|
| 81 |
+
"style-transfer" in model_lower,
|
| 82 |
+
]
|
| 83 |
+
):
|
| 84 |
+
return True
|
| 85 |
+
return False
|
| 86 |
+
|
| 87 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 88 |
+
"""
|
| 89 |
+
Return list of OpenAI params supported by Bedrock Stability.
|
| 90 |
+
"""
|
| 91 |
+
return [
|
| 92 |
+
"n", # Number of images (Stability always returns 1, we can loop)
|
| 93 |
+
"size", # Maps to aspect_ratio
|
| 94 |
+
"response_format", # b64_json or url (Stability only returns b64)
|
| 95 |
+
"mask",
|
| 96 |
+
]
|
| 97 |
+
|
| 98 |
+
def map_openai_params(
|
| 99 |
+
self,
|
| 100 |
+
image_edit_optional_params: ImageEditOptionalRequestParams,
|
| 101 |
+
model: str,
|
| 102 |
+
drop_params: bool,
|
| 103 |
+
) -> Dict:
|
| 104 |
+
"""
|
| 105 |
+
Map OpenAI parameters to Bedrock Stability parameters.
|
| 106 |
+
|
| 107 |
+
OpenAI -> Stability mappings:
|
| 108 |
+
- size -> aspect_ratio
|
| 109 |
+
- n -> (handled separately, Stability returns 1 image per request)
|
| 110 |
+
"""
|
| 111 |
+
supported_params = self.get_supported_openai_params(model)
|
| 112 |
+
# Define mapping from OpenAI params to Stability params
|
| 113 |
+
param_mapping = {
|
| 114 |
+
"size": "aspect_ratio",
|
| 115 |
+
# "n" and "response_format" are handled separately
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
# Create a copy to not mutate original - convert TypedDict to regular dict
|
| 119 |
+
mapped_params: Dict[str, Any] = dict(image_edit_optional_params)
|
| 120 |
+
|
| 121 |
+
for k, v in image_edit_optional_params.items():
|
| 122 |
+
if k in param_mapping:
|
| 123 |
+
# Map param if mapping exists and value is valid
|
| 124 |
+
if k == "size" and v in OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO:
|
| 125 |
+
mapped_params[param_mapping[k]] = OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO[v] # type: ignore
|
| 126 |
+
# Don't copy "size" itself to final dict
|
| 127 |
+
elif k == "n":
|
| 128 |
+
# Store for logic but do not add to outgoing params
|
| 129 |
+
mapped_params["_n"] = v
|
| 130 |
+
elif k == "response_format":
|
| 131 |
+
# Only b64 supported at Stability; store for postprocessing
|
| 132 |
+
mapped_params["_response_format"] = v
|
| 133 |
+
elif k not in supported_params:
|
| 134 |
+
if not drop_params:
|
| 135 |
+
raise ValueError(
|
| 136 |
+
f"Parameter {k} is not supported for model {model}. "
|
| 137 |
+
f"Supported parameters are {supported_params}. "
|
| 138 |
+
f"Set drop_params=True to drop unsupported parameters."
|
| 139 |
+
)
|
| 140 |
+
# Otherwise, param will simply be dropped
|
| 141 |
+
else:
|
| 142 |
+
# param is supported and not mapped, keep as-is
|
| 143 |
+
continue
|
| 144 |
+
|
| 145 |
+
# Remove OpenAI params that have been mapped unless they're in stability
|
| 146 |
+
for mapped in ["size", "n", "response_format"]:
|
| 147 |
+
if mapped in mapped_params:
|
| 148 |
+
del mapped_params[mapped]
|
| 149 |
+
|
| 150 |
+
return mapped_params
|
| 151 |
+
|
| 152 |
+
def transform_image_edit_request( # noqa: PLR0915
|
| 153 |
+
self,
|
| 154 |
+
model: str,
|
| 155 |
+
prompt: Optional[str],
|
| 156 |
+
image: Optional[FileTypes],
|
| 157 |
+
image_edit_optional_request_params: Dict,
|
| 158 |
+
litellm_params: GenericLiteLLMParams,
|
| 159 |
+
headers: dict,
|
| 160 |
+
) -> Tuple[Dict, Any]:
|
| 161 |
+
"""
|
| 162 |
+
Transform OpenAI-style request to Bedrock Stability request format.
|
| 163 |
+
|
| 164 |
+
Returns the request body dict that will be JSON-encoded by the handler.
|
| 165 |
+
"""
|
| 166 |
+
# Build Bedrock Stability request
|
| 167 |
+
data: Dict[str, Any] = {
|
| 168 |
+
"output_format": "png", # Default to PNG
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
# Add prompt only if provided (some models don't require it)
|
| 172 |
+
if prompt is not None and prompt != "":
|
| 173 |
+
data["prompt"] = prompt
|
| 174 |
+
|
| 175 |
+
# Convert image to base64 if provided
|
| 176 |
+
if image is not None:
|
| 177 |
+
image_b64: str
|
| 178 |
+
if hasattr(image, "read") and callable(getattr(image, "read", None)):
|
| 179 |
+
# File-like object (e.g., BufferedReader from open())
|
| 180 |
+
image_bytes = image.read() # type: ignore
|
| 181 |
+
image_b64 = base64.b64encode(image_bytes).decode("utf-8") # type: ignore
|
| 182 |
+
elif isinstance(image, bytes):
|
| 183 |
+
# Raw bytes
|
| 184 |
+
image_b64 = base64.b64encode(image).decode("utf-8")
|
| 185 |
+
elif isinstance(image, str):
|
| 186 |
+
# Already a base64 string
|
| 187 |
+
image_b64 = image
|
| 188 |
+
else:
|
| 189 |
+
# Try to handle as bytes
|
| 190 |
+
image_b64 = base64.b64encode(bytes(image)).decode("utf-8") # type: ignore
|
| 191 |
+
|
| 192 |
+
# For style-transfer models, map image to init_image
|
| 193 |
+
model_lower = model.lower()
|
| 194 |
+
if "style-transfer" in model_lower:
|
| 195 |
+
data["init_image"] = image_b64
|
| 196 |
+
else:
|
| 197 |
+
data["image"] = image_b64
|
| 198 |
+
|
| 199 |
+
# Add optional params (already mapped in map_openai_params)
|
| 200 |
+
for key, value in image_edit_optional_request_params.items(): # type: ignore
|
| 201 |
+
# Skip internal params (prefixed with _)
|
| 202 |
+
if key.startswith("_") or value is None:
|
| 203 |
+
continue
|
| 204 |
+
|
| 205 |
+
# File-like optional params (mask, init_image, style_image, etc.)
|
| 206 |
+
if key in ["mask", "init_image", "style_image"]:
|
| 207 |
+
# Handle case where value might be in a list
|
| 208 |
+
file_value = value
|
| 209 |
+
if isinstance(value, list) and len(value) > 0:
|
| 210 |
+
file_value = value[0]
|
| 211 |
+
|
| 212 |
+
if hasattr(file_value, "read") and callable(
|
| 213 |
+
getattr(file_value, "read", None)
|
| 214 |
+
):
|
| 215 |
+
file_bytes = file_value.read() # type: ignore
|
| 216 |
+
elif isinstance(file_value, bytes):
|
| 217 |
+
file_bytes = file_value
|
| 218 |
+
elif isinstance(file_value, str):
|
| 219 |
+
# Already a base64 string
|
| 220 |
+
data[key] = file_value
|
| 221 |
+
continue
|
| 222 |
+
else:
|
| 223 |
+
file_bytes = file_value # type: ignore
|
| 224 |
+
|
| 225 |
+
if isinstance(file_bytes, bytes):
|
| 226 |
+
file_b64 = base64.b64encode(file_bytes).decode("utf-8")
|
| 227 |
+
else:
|
| 228 |
+
file_b64 = str(file_bytes)
|
| 229 |
+
data[key] = file_b64
|
| 230 |
+
continue
|
| 231 |
+
|
| 232 |
+
# Numeric fields that need to be converted to int/float
|
| 233 |
+
numeric_int_fields = ["left", "right", "up", "down", "seed"]
|
| 234 |
+
numeric_float_fields = [
|
| 235 |
+
"strength",
|
| 236 |
+
"creativity",
|
| 237 |
+
"control_strength",
|
| 238 |
+
"grow_mask",
|
| 239 |
+
"fidelity",
|
| 240 |
+
"composition_fidelity",
|
| 241 |
+
"style_strength",
|
| 242 |
+
"change_strength",
|
| 243 |
+
]
|
| 244 |
+
|
| 245 |
+
if key in numeric_int_fields:
|
| 246 |
+
# Convert to int (these are pixel values for outpaint)
|
| 247 |
+
try:
|
| 248 |
+
data[key] = int(value) # type: ignore
|
| 249 |
+
except (ValueError, TypeError):
|
| 250 |
+
data[key] = value # type: ignore
|
| 251 |
+
elif key in numeric_float_fields:
|
| 252 |
+
# Convert to float
|
| 253 |
+
try:
|
| 254 |
+
data[key] = float(value) # type: ignore
|
| 255 |
+
except (ValueError, TypeError):
|
| 256 |
+
data[key] = value # type: ignore
|
| 257 |
+
|
| 258 |
+
# Supported text fields
|
| 259 |
+
elif key in [
|
| 260 |
+
"negative_prompt",
|
| 261 |
+
"aspect_ratio",
|
| 262 |
+
"output_format",
|
| 263 |
+
"model",
|
| 264 |
+
"mode",
|
| 265 |
+
"style_preset",
|
| 266 |
+
"select_prompt",
|
| 267 |
+
"search_prompt",
|
| 268 |
+
]:
|
| 269 |
+
data[key] = value # type: ignore
|
| 270 |
+
|
| 271 |
+
return data, {}
|
| 272 |
+
|
| 273 |
+
def transform_image_edit_response(
|
| 274 |
+
self,
|
| 275 |
+
model: str,
|
| 276 |
+
raw_response: httpx.Response,
|
| 277 |
+
logging_obj: LiteLLMLoggingObj,
|
| 278 |
+
api_key: Optional[str] = None,
|
| 279 |
+
json_mode: Optional[bool] = None,
|
| 280 |
+
) -> ImageResponse:
|
| 281 |
+
"""
|
| 282 |
+
Transform Bedrock Stability response to OpenAI-compatible ImageResponse.
|
| 283 |
+
|
| 284 |
+
Bedrock returns: {"images": ["base64..."], "finish_reasons": [null], "seeds": [123]}
|
| 285 |
+
OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp}
|
| 286 |
+
"""
|
| 287 |
+
try:
|
| 288 |
+
response_data = raw_response.json()
|
| 289 |
+
except Exception as e:
|
| 290 |
+
raise self.get_error_class(
|
| 291 |
+
error_message=f"Error parsing Bedrock Stability response: {e}",
|
| 292 |
+
status_code=raw_response.status_code,
|
| 293 |
+
headers=raw_response.headers,
|
| 294 |
+
)
|
| 295 |
+
|
| 296 |
+
# Check for errors in response
|
| 297 |
+
if "errors" in response_data:
|
| 298 |
+
raise self.get_error_class(
|
| 299 |
+
error_message=f"Bedrock Stability error: {response_data['errors']}",
|
| 300 |
+
status_code=raw_response.status_code,
|
| 301 |
+
headers=raw_response.headers,
|
| 302 |
+
)
|
| 303 |
+
|
| 304 |
+
# Check finish_reasons
|
| 305 |
+
finish_reasons = response_data.get("finish_reasons", [])
|
| 306 |
+
if finish_reasons and finish_reasons[0]:
|
| 307 |
+
raise self.get_error_class(
|
| 308 |
+
error_message=f"Bedrock Stability error: {finish_reasons[0]}",
|
| 309 |
+
status_code=400,
|
| 310 |
+
headers=raw_response.headers,
|
| 311 |
+
)
|
| 312 |
+
|
| 313 |
+
model_response = ImageResponse()
|
| 314 |
+
if not model_response.data:
|
| 315 |
+
model_response.data = []
|
| 316 |
+
|
| 317 |
+
# Extract images from response
|
| 318 |
+
images = response_data.get("images", [])
|
| 319 |
+
if images:
|
| 320 |
+
for image_b64 in images:
|
| 321 |
+
if image_b64:
|
| 322 |
+
model_response.data.append(
|
| 323 |
+
ImageObject(
|
| 324 |
+
b64_json=image_b64,
|
| 325 |
+
url=None,
|
| 326 |
+
revised_prompt=None,
|
| 327 |
+
)
|
| 328 |
+
)
|
| 329 |
+
|
| 330 |
+
if not hasattr(model_response, "_hidden_params"):
|
| 331 |
+
model_response._hidden_params = {}
|
| 332 |
+
if "additional_headers" not in model_response._hidden_params:
|
| 333 |
+
model_response._hidden_params["additional_headers"] = {}
|
| 334 |
+
|
| 335 |
+
# Set cost based on model
|
| 336 |
+
model_info = get_model_info(model, custom_llm_provider="bedrock")
|
| 337 |
+
cost_per_image = model_info.get("output_cost_per_image", 0)
|
| 338 |
+
if cost_per_image is not None:
|
| 339 |
+
model_response._hidden_params["additional_headers"][
|
| 340 |
+
"llm_provider-x-litellm-response-cost"
|
| 341 |
+
] = float(cost_per_image)
|
| 342 |
+
|
| 343 |
+
return model_response
|
| 344 |
+
|
| 345 |
+
def use_multipart_form_data(self) -> bool:
|
| 346 |
+
"""
|
| 347 |
+
Bedrock Stability uses JSON format, not multipart/form-data.
|
| 348 |
+
"""
|
| 349 |
+
return False
|
| 350 |
+
|
| 351 |
+
def get_complete_url(
|
| 352 |
+
self,
|
| 353 |
+
model: str,
|
| 354 |
+
api_base: Optional[str],
|
| 355 |
+
litellm_params: dict,
|
| 356 |
+
) -> str:
|
| 357 |
+
"""
|
| 358 |
+
Get the complete URL for the Bedrock Image Edit API.
|
| 359 |
+
|
| 360 |
+
For Bedrock, this is handled by the handler which constructs the endpoint URL
|
| 361 |
+
based on the model ID and AWS region. This method is required by the base class
|
| 362 |
+
but the actual URL construction happens in BedrockImageEdit.image_edit().
|
| 363 |
+
|
| 364 |
+
Returns a placeholder - the real endpoint is constructed in the handler.
|
| 365 |
+
"""
|
| 366 |
+
# Bedrock URLs are constructed in the handler using boto3
|
| 367 |
+
# This is a placeholder for the abstract method requirement
|
| 368 |
+
return "bedrock://image-edit"
|
| 369 |
+
|
| 370 |
+
def validate_environment(
|
| 371 |
+
self,
|
| 372 |
+
headers: dict,
|
| 373 |
+
model: str,
|
| 374 |
+
api_key: Optional[str] = None,
|
| 375 |
+
) -> dict:
|
| 376 |
+
"""
|
| 377 |
+
Validate environment for Bedrock Stability image edit.
|
| 378 |
+
|
| 379 |
+
For Bedrock, AWS credentials are managed by the BaseAWSLLM class.
|
| 380 |
+
This method validates that headers are properly set up.
|
| 381 |
+
|
| 382 |
+
Args:
|
| 383 |
+
headers: The request headers to validate/update
|
| 384 |
+
model: The model name being used
|
| 385 |
+
api_key: Optional API key (not used for Bedrock, which uses AWS credentials)
|
| 386 |
+
|
| 387 |
+
Returns:
|
| 388 |
+
Updated headers dict
|
| 389 |
+
"""
|
| 390 |
+
if headers is None:
|
| 391 |
+
headers = {}
|
| 392 |
+
|
| 393 |
+
# Bedrock uses AWS credentials, not API keys
|
| 394 |
+
# Headers are set up by the handler's get_request_headers() method
|
| 395 |
+
# This just ensures basic headers are present
|
| 396 |
+
if "Content-Type" not in headers:
|
| 397 |
+
headers["Content-Type"] = "application/json"
|
| 398 |
+
|
| 399 |
+
return headers
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import types
|
| 2 |
+
from typing import Any, Dict, List, Optional
|
| 3 |
+
|
| 4 |
+
from openai.types.image import Image
|
| 5 |
+
|
| 6 |
+
from litellm.types.llms.bedrock import (
|
| 7 |
+
AmazonNovaCanvasColorGuidedGenerationParams,
|
| 8 |
+
AmazonNovaCanvasColorGuidedRequest,
|
| 9 |
+
AmazonNovaCanvasImageGenerationConfig,
|
| 10 |
+
AmazonNovaCanvasInpaintingParams,
|
| 11 |
+
AmazonNovaCanvasInpaintingRequest,
|
| 12 |
+
AmazonNovaCanvasRequestBase,
|
| 13 |
+
AmazonNovaCanvasTextToImageParams,
|
| 14 |
+
AmazonNovaCanvasTextToImageRequest,
|
| 15 |
+
AmazonNovaCanvasTextToImageResponse,
|
| 16 |
+
)
|
| 17 |
+
from litellm.llms.bedrock.common_utils import get_cached_model_info
|
| 18 |
+
from litellm.types.utils import ImageResponse
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class AmazonNovaCanvasConfig:
|
| 22 |
+
"""
|
| 23 |
+
Reference: https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/model-catalog/serverless/amazon.nova-canvas-v1:0
|
| 24 |
+
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
@classmethod
|
| 28 |
+
def get_config(cls):
|
| 29 |
+
return {
|
| 30 |
+
k: v
|
| 31 |
+
for k, v in cls.__dict__.items()
|
| 32 |
+
if not k.startswith("__")
|
| 33 |
+
and not isinstance(
|
| 34 |
+
v,
|
| 35 |
+
(
|
| 36 |
+
types.FunctionType,
|
| 37 |
+
types.BuiltinFunctionType,
|
| 38 |
+
classmethod,
|
| 39 |
+
staticmethod,
|
| 40 |
+
),
|
| 41 |
+
)
|
| 42 |
+
and v is not None
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
@classmethod
|
| 46 |
+
def get_supported_openai_params(cls, model: Optional[str] = None) -> List:
|
| 47 |
+
""" """
|
| 48 |
+
return ["n", "size", "quality"]
|
| 49 |
+
|
| 50 |
+
@classmethod
|
| 51 |
+
def _is_nova_model(cls, model: Optional[str] = None) -> bool:
|
| 52 |
+
"""
|
| 53 |
+
Returns True if the model is a Nova Canvas model
|
| 54 |
+
|
| 55 |
+
Nova models follow this pattern:
|
| 56 |
+
|
| 57 |
+
"""
|
| 58 |
+
if model and "amazon.nova-canvas" in model:
|
| 59 |
+
return True
|
| 60 |
+
return False
|
| 61 |
+
|
| 62 |
+
@classmethod
|
| 63 |
+
def transform_request_body(
|
| 64 |
+
cls, text: str, optional_params: dict
|
| 65 |
+
) -> AmazonNovaCanvasRequestBase:
|
| 66 |
+
"""
|
| 67 |
+
Transform the request body for Amazon Nova Canvas model
|
| 68 |
+
"""
|
| 69 |
+
task_type = optional_params.pop("taskType", "TEXT_IMAGE")
|
| 70 |
+
image_generation_config = optional_params.pop("imageGenerationConfig", {})
|
| 71 |
+
|
| 72 |
+
# Extract model_id parameter to prevent "extraneous key" error from Bedrock API
|
| 73 |
+
# Following the same pattern as chat completions and embeddings
|
| 74 |
+
unencoded_model_id = optional_params.pop("model_id", None) # noqa: F841
|
| 75 |
+
|
| 76 |
+
image_generation_config = {**image_generation_config, **optional_params}
|
| 77 |
+
if task_type == "TEXT_IMAGE":
|
| 78 |
+
text_to_image_params: Dict[str, Any] = image_generation_config.pop(
|
| 79 |
+
"textToImageParams", {}
|
| 80 |
+
)
|
| 81 |
+
text_to_image_params = {"text": text, **text_to_image_params}
|
| 82 |
+
try:
|
| 83 |
+
text_to_image_params_typed = AmazonNovaCanvasTextToImageParams(
|
| 84 |
+
**text_to_image_params # type: ignore
|
| 85 |
+
)
|
| 86 |
+
except Exception as e:
|
| 87 |
+
raise ValueError(
|
| 88 |
+
f"Error transforming text to image params: {e}. Got params: {text_to_image_params}, Expected params: {AmazonNovaCanvasTextToImageParams.__annotations__}"
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
try:
|
| 92 |
+
image_generation_config_typed = AmazonNovaCanvasImageGenerationConfig(
|
| 93 |
+
**image_generation_config
|
| 94 |
+
)
|
| 95 |
+
except Exception as e:
|
| 96 |
+
raise ValueError(
|
| 97 |
+
f"Error transforming image generation config: {e}. Got params: {image_generation_config}, Expected params: {AmazonNovaCanvasImageGenerationConfig.__annotations__}"
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
return AmazonNovaCanvasTextToImageRequest(
|
| 101 |
+
textToImageParams=text_to_image_params_typed,
|
| 102 |
+
taskType=task_type,
|
| 103 |
+
imageGenerationConfig=image_generation_config_typed,
|
| 104 |
+
)
|
| 105 |
+
if task_type == "COLOR_GUIDED_GENERATION":
|
| 106 |
+
color_guided_generation_params: Dict[
|
| 107 |
+
str, Any
|
| 108 |
+
] = image_generation_config.pop("colorGuidedGenerationParams", {})
|
| 109 |
+
color_guided_generation_params = {
|
| 110 |
+
"text": text,
|
| 111 |
+
**color_guided_generation_params,
|
| 112 |
+
}
|
| 113 |
+
try:
|
| 114 |
+
color_guided_generation_params_typed = AmazonNovaCanvasColorGuidedGenerationParams(
|
| 115 |
+
**color_guided_generation_params # type: ignore
|
| 116 |
+
)
|
| 117 |
+
except Exception as e:
|
| 118 |
+
raise ValueError(
|
| 119 |
+
f"Error transforming color guided generation params: {e}. Got params: {color_guided_generation_params}, Expected params: {AmazonNovaCanvasColorGuidedGenerationParams.__annotations__}"
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
try:
|
| 123 |
+
image_generation_config_typed = AmazonNovaCanvasImageGenerationConfig(
|
| 124 |
+
**image_generation_config
|
| 125 |
+
)
|
| 126 |
+
except Exception as e:
|
| 127 |
+
raise ValueError(
|
| 128 |
+
f"Error transforming image generation config: {e}. Got params: {image_generation_config}, Expected params: {AmazonNovaCanvasImageGenerationConfig.__annotations__}"
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
return AmazonNovaCanvasColorGuidedRequest(
|
| 132 |
+
taskType=task_type,
|
| 133 |
+
colorGuidedGenerationParams=color_guided_generation_params_typed,
|
| 134 |
+
imageGenerationConfig=image_generation_config_typed,
|
| 135 |
+
)
|
| 136 |
+
if task_type == "INPAINTING":
|
| 137 |
+
inpainting_params: Dict[str, Any] = image_generation_config.pop(
|
| 138 |
+
"inpaintingParams", {}
|
| 139 |
+
)
|
| 140 |
+
inpainting_params = {"text": text, **inpainting_params}
|
| 141 |
+
try:
|
| 142 |
+
inpainting_params_typed = AmazonNovaCanvasInpaintingParams(
|
| 143 |
+
**inpainting_params # type: ignore
|
| 144 |
+
)
|
| 145 |
+
except Exception as e:
|
| 146 |
+
raise ValueError(
|
| 147 |
+
f"Error transforming inpainting params: {e}. Got params: {inpainting_params}, Expected params: {AmazonNovaCanvasInpaintingParams.__annotations__}"
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
try:
|
| 151 |
+
image_generation_config_typed = AmazonNovaCanvasImageGenerationConfig(
|
| 152 |
+
**image_generation_config
|
| 153 |
+
)
|
| 154 |
+
except Exception as e:
|
| 155 |
+
raise ValueError(
|
| 156 |
+
f"Error transforming image generation config: {e}. Got params: {image_generation_config}, Expected params: {AmazonNovaCanvasImageGenerationConfig.__annotations__}"
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
return AmazonNovaCanvasInpaintingRequest(
|
| 160 |
+
taskType=task_type,
|
| 161 |
+
inpaintingParams=inpainting_params_typed,
|
| 162 |
+
imageGenerationConfig=image_generation_config_typed,
|
| 163 |
+
)
|
| 164 |
+
raise NotImplementedError(f"Task type {task_type} is not supported")
|
| 165 |
+
|
| 166 |
+
@classmethod
|
| 167 |
+
def map_openai_params(cls, non_default_params: dict, optional_params: dict) -> dict:
|
| 168 |
+
"""
|
| 169 |
+
Map the OpenAI params to the Bedrock params
|
| 170 |
+
"""
|
| 171 |
+
_size = non_default_params.get("size")
|
| 172 |
+
if _size is not None:
|
| 173 |
+
width, height = _size.split("x")
|
| 174 |
+
optional_params["width"], optional_params["height"] = int(width), int(
|
| 175 |
+
height
|
| 176 |
+
)
|
| 177 |
+
if non_default_params.get("n") is not None:
|
| 178 |
+
optional_params["numberOfImages"] = non_default_params.get("n")
|
| 179 |
+
if non_default_params.get("quality") is not None:
|
| 180 |
+
if non_default_params.get("quality") in ("hd", "premium"):
|
| 181 |
+
optional_params["quality"] = "premium"
|
| 182 |
+
if non_default_params.get("quality") == "standard":
|
| 183 |
+
optional_params["quality"] = "standard"
|
| 184 |
+
return optional_params
|
| 185 |
+
|
| 186 |
+
@classmethod
|
| 187 |
+
def transform_response_dict_to_openai_response(
|
| 188 |
+
cls, model_response: ImageResponse, response_dict: dict
|
| 189 |
+
) -> ImageResponse:
|
| 190 |
+
"""
|
| 191 |
+
Transform the response dict to the OpenAI response
|
| 192 |
+
"""
|
| 193 |
+
|
| 194 |
+
nova_response = AmazonNovaCanvasTextToImageResponse(**response_dict)
|
| 195 |
+
openai_images: List[Image] = []
|
| 196 |
+
for _img in nova_response.get("images", []):
|
| 197 |
+
openai_images.append(Image(b64_json=_img))
|
| 198 |
+
|
| 199 |
+
model_response.data = openai_images
|
| 200 |
+
return model_response
|
| 201 |
+
|
| 202 |
+
@classmethod
|
| 203 |
+
def cost_calculator(
|
| 204 |
+
cls,
|
| 205 |
+
model: str,
|
| 206 |
+
image_response: ImageResponse,
|
| 207 |
+
size: Optional[str] = None,
|
| 208 |
+
optional_params: Optional[dict] = None,
|
| 209 |
+
) -> float:
|
| 210 |
+
get_model_info = get_cached_model_info()
|
| 211 |
+
model_info = get_model_info(
|
| 212 |
+
model=model,
|
| 213 |
+
custom_llm_provider="bedrock",
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
|
| 217 |
+
num_images: int = 0
|
| 218 |
+
if image_response.data:
|
| 219 |
+
num_images = len(image_response.data)
|
| 220 |
+
return output_cost_per_image * num_images
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import os
|
| 3 |
+
import types
|
| 4 |
+
from typing import List, Optional
|
| 5 |
+
|
| 6 |
+
from openai.types.image import Image
|
| 7 |
+
|
| 8 |
+
from litellm.llms.bedrock.common_utils import get_cached_model_info
|
| 9 |
+
from litellm.types.utils import ImageResponse
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AmazonStabilityConfig:
|
| 13 |
+
"""
|
| 14 |
+
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=stability.stable-diffusion-xl-v0
|
| 15 |
+
|
| 16 |
+
Supported Params for the Amazon / Stable Diffusion models:
|
| 17 |
+
|
| 18 |
+
- `cfg_scale` (integer): Default `7`. Between [ 0 .. 35 ]. How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt)
|
| 19 |
+
|
| 20 |
+
- `seed` (float): Default: `0`. Between [ 0 .. 4294967295 ]. Random noise seed (omit this option or use 0 for a random seed)
|
| 21 |
+
|
| 22 |
+
- `steps` (array of strings): Default `30`. Between [ 10 .. 50 ]. Number of diffusion steps to run.
|
| 23 |
+
|
| 24 |
+
- `width` (integer): Default: `512`. multiple of 64 >= 128. Width of the image to generate, in pixels, in an increment divible by 64.
|
| 25 |
+
Engine-specific dimension validation:
|
| 26 |
+
|
| 27 |
+
- SDXL Beta: must be between 128x128 and 512x896 (or 896x512); only one dimension can be greater than 512.
|
| 28 |
+
- SDXL v0.9: must be one of 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640, 640x1536, 768x1344, 832x1216, or 896x1152
|
| 29 |
+
- SDXL v1.0: same as SDXL v0.9
|
| 30 |
+
- SD v1.6: must be between 320x320 and 1536x1536
|
| 31 |
+
|
| 32 |
+
- `height` (integer): Default: `512`. multiple of 64 >= 128. Height of the image to generate, in pixels, in an increment divible by 64.
|
| 33 |
+
Engine-specific dimension validation:
|
| 34 |
+
|
| 35 |
+
- SDXL Beta: must be between 128x128 and 512x896 (or 896x512); only one dimension can be greater than 512.
|
| 36 |
+
- SDXL v0.9: must be one of 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640, 640x1536, 768x1344, 832x1216, or 896x1152
|
| 37 |
+
- SDXL v1.0: same as SDXL v0.9
|
| 38 |
+
- SD v1.6: must be between 320x320 and 1536x1536
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
cfg_scale: Optional[int] = None
|
| 42 |
+
seed: Optional[float] = None
|
| 43 |
+
steps: Optional[List[str]] = None
|
| 44 |
+
width: Optional[int] = None
|
| 45 |
+
height: Optional[int] = None
|
| 46 |
+
|
| 47 |
+
def __init__(
|
| 48 |
+
self,
|
| 49 |
+
cfg_scale: Optional[int] = None,
|
| 50 |
+
seed: Optional[float] = None,
|
| 51 |
+
steps: Optional[List[str]] = None,
|
| 52 |
+
width: Optional[int] = None,
|
| 53 |
+
height: Optional[int] = None,
|
| 54 |
+
) -> None:
|
| 55 |
+
locals_ = locals().copy()
|
| 56 |
+
for key, value in locals_.items():
|
| 57 |
+
if key != "self" and value is not None:
|
| 58 |
+
setattr(self.__class__, key, value)
|
| 59 |
+
|
| 60 |
+
@classmethod
|
| 61 |
+
def get_config(cls):
|
| 62 |
+
return {
|
| 63 |
+
k: v
|
| 64 |
+
for k, v in cls.__dict__.items()
|
| 65 |
+
if not k.startswith("__")
|
| 66 |
+
and not isinstance(
|
| 67 |
+
v,
|
| 68 |
+
(
|
| 69 |
+
types.FunctionType,
|
| 70 |
+
types.BuiltinFunctionType,
|
| 71 |
+
classmethod,
|
| 72 |
+
staticmethod,
|
| 73 |
+
),
|
| 74 |
+
)
|
| 75 |
+
and v is not None
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
@classmethod
|
| 79 |
+
def get_supported_openai_params(cls, model: Optional[str] = None) -> List:
|
| 80 |
+
return ["size"]
|
| 81 |
+
|
| 82 |
+
@classmethod
|
| 83 |
+
def map_openai_params(
|
| 84 |
+
cls,
|
| 85 |
+
non_default_params: dict,
|
| 86 |
+
optional_params: dict,
|
| 87 |
+
):
|
| 88 |
+
_size = non_default_params.get("size")
|
| 89 |
+
if _size is not None:
|
| 90 |
+
width, height = _size.split("x")
|
| 91 |
+
optional_params["width"] = int(width)
|
| 92 |
+
optional_params["height"] = int(height)
|
| 93 |
+
|
| 94 |
+
return optional_params
|
| 95 |
+
|
| 96 |
+
@classmethod
|
| 97 |
+
def transform_request_body(
|
| 98 |
+
cls,
|
| 99 |
+
text: str,
|
| 100 |
+
optional_params: dict,
|
| 101 |
+
) -> dict:
|
| 102 |
+
inference_params = copy.deepcopy(optional_params)
|
| 103 |
+
inference_params.pop(
|
| 104 |
+
"user", None
|
| 105 |
+
) # make sure user is not passed in for bedrock call
|
| 106 |
+
|
| 107 |
+
prompt = text.replace(os.linesep, " ")
|
| 108 |
+
## LOAD CONFIG
|
| 109 |
+
config = cls.get_config()
|
| 110 |
+
for k, v in config.items():
|
| 111 |
+
if (
|
| 112 |
+
k not in inference_params
|
| 113 |
+
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 114 |
+
inference_params[k] = v
|
| 115 |
+
|
| 116 |
+
return {
|
| 117 |
+
"text_prompts": [{"text": prompt, "weight": 1}],
|
| 118 |
+
**inference_params,
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
@classmethod
|
| 122 |
+
def transform_response_dict_to_openai_response(
|
| 123 |
+
cls, model_response: ImageResponse, response_dict: dict
|
| 124 |
+
) -> ImageResponse:
|
| 125 |
+
image_list: List[Image] = []
|
| 126 |
+
for artifact in response_dict["artifacts"]:
|
| 127 |
+
_image = Image(b64_json=artifact["base64"])
|
| 128 |
+
image_list.append(_image)
|
| 129 |
+
|
| 130 |
+
model_response.data = image_list
|
| 131 |
+
|
| 132 |
+
return model_response
|
| 133 |
+
|
| 134 |
+
@classmethod
|
| 135 |
+
def cost_calculator(
|
| 136 |
+
cls,
|
| 137 |
+
model: str,
|
| 138 |
+
image_response: ImageResponse,
|
| 139 |
+
size: Optional[str] = None,
|
| 140 |
+
optional_params: Optional[dict] = None,
|
| 141 |
+
) -> float:
|
| 142 |
+
optional_params = optional_params or {}
|
| 143 |
+
|
| 144 |
+
# see model_prices_and_context_window.json for details on how steps is used
|
| 145 |
+
# Reference pricing by steps for stability 1: https://aws.amazon.com/bedrock/pricing/
|
| 146 |
+
_steps = optional_params.get("steps", 50)
|
| 147 |
+
steps = "max-steps" if _steps > 50 else "50-steps"
|
| 148 |
+
|
| 149 |
+
# size is stored in model_prices_and_context_window.json as 1024-x-1024
|
| 150 |
+
# current size has 1024x1024
|
| 151 |
+
size = size or "1024-x-1024"
|
| 152 |
+
model = f"{size}/{steps}/{model}"
|
| 153 |
+
|
| 154 |
+
get_model_info = get_cached_model_info()
|
| 155 |
+
model_info = get_model_info(
|
| 156 |
+
model=model,
|
| 157 |
+
custom_llm_provider="bedrock",
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
|
| 161 |
+
num_images: int = 0
|
| 162 |
+
if image_response.data:
|
| 163 |
+
num_images = len(image_response.data)
|
| 164 |
+
return output_cost_per_image * num_images
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import types
|
| 2 |
+
from typing import List, Optional
|
| 3 |
+
|
| 4 |
+
from openai.types.image import Image
|
| 5 |
+
|
| 6 |
+
from litellm.llms.bedrock.common_utils import BedrockError
|
| 7 |
+
from litellm.types.llms.bedrock import (
|
| 8 |
+
AmazonStability3TextToImageRequest,
|
| 9 |
+
AmazonStability3TextToImageResponse,
|
| 10 |
+
)
|
| 11 |
+
from litellm.llms.bedrock.common_utils import get_cached_model_info
|
| 12 |
+
from litellm.types.utils import ImageResponse
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class AmazonStability3Config:
|
| 16 |
+
"""
|
| 17 |
+
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=stability.stable-diffusion-xl-v0
|
| 18 |
+
|
| 19 |
+
Stability API Ref: https://platform.stability.ai/docs/api-reference#tag/Generate/paths/~1v2beta~1stable-image~1generate~1sd3/post
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
@classmethod
|
| 23 |
+
def get_config(cls):
|
| 24 |
+
return {
|
| 25 |
+
k: v
|
| 26 |
+
for k, v in cls.__dict__.items()
|
| 27 |
+
if not k.startswith("__")
|
| 28 |
+
and not isinstance(
|
| 29 |
+
v,
|
| 30 |
+
(
|
| 31 |
+
types.FunctionType,
|
| 32 |
+
types.BuiltinFunctionType,
|
| 33 |
+
classmethod,
|
| 34 |
+
staticmethod,
|
| 35 |
+
),
|
| 36 |
+
)
|
| 37 |
+
and v is not None
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
@classmethod
|
| 41 |
+
def get_supported_openai_params(cls, model: Optional[str] = None) -> List:
|
| 42 |
+
"""
|
| 43 |
+
No additional OpenAI params are mapped for stability 3
|
| 44 |
+
"""
|
| 45 |
+
return []
|
| 46 |
+
|
| 47 |
+
@classmethod
|
| 48 |
+
def _is_stability_3_model(cls, model: Optional[str] = None) -> bool:
|
| 49 |
+
"""
|
| 50 |
+
Returns True if the model is a Stability 3 model
|
| 51 |
+
|
| 52 |
+
Stability 3 models follow this pattern:
|
| 53 |
+
sd3-large
|
| 54 |
+
sd3-large-turbo
|
| 55 |
+
sd3-medium
|
| 56 |
+
sd3.5-large
|
| 57 |
+
sd3.5-large-turbo
|
| 58 |
+
|
| 59 |
+
Stability ultra models
|
| 60 |
+
stable-image-ultra-v1
|
| 61 |
+
"""
|
| 62 |
+
if model:
|
| 63 |
+
if "sd3" in model or "sd3.5" in model:
|
| 64 |
+
return True
|
| 65 |
+
if "stable-image" in model:
|
| 66 |
+
return True
|
| 67 |
+
return False
|
| 68 |
+
|
| 69 |
+
@classmethod
|
| 70 |
+
def transform_request_body(
|
| 71 |
+
cls, text: str, optional_params: dict
|
| 72 |
+
) -> AmazonStability3TextToImageRequest:
|
| 73 |
+
"""
|
| 74 |
+
Transform the request body for the Stability 3 models
|
| 75 |
+
"""
|
| 76 |
+
data = AmazonStability3TextToImageRequest(prompt=text, **optional_params)
|
| 77 |
+
return data
|
| 78 |
+
|
| 79 |
+
@classmethod
|
| 80 |
+
def map_openai_params(cls, non_default_params: dict, optional_params: dict) -> dict:
|
| 81 |
+
"""
|
| 82 |
+
Map the OpenAI params to the Bedrock params
|
| 83 |
+
|
| 84 |
+
No OpenAI params are mapped for Stability 3, so directly return the optional_params
|
| 85 |
+
"""
|
| 86 |
+
return optional_params
|
| 87 |
+
|
| 88 |
+
@classmethod
|
| 89 |
+
def transform_response_dict_to_openai_response(
|
| 90 |
+
cls, model_response: ImageResponse, response_dict: dict
|
| 91 |
+
) -> ImageResponse:
|
| 92 |
+
"""
|
| 93 |
+
Transform the response dict to the OpenAI response
|
| 94 |
+
"""
|
| 95 |
+
|
| 96 |
+
stability_3_response = AmazonStability3TextToImageResponse(**response_dict)
|
| 97 |
+
|
| 98 |
+
finish_reasons = stability_3_response.get("finish_reasons", [])
|
| 99 |
+
finish_reasons = [reason for reason in finish_reasons if reason]
|
| 100 |
+
if len(finish_reasons) > 0:
|
| 101 |
+
raise BedrockError(status_code=400, message="; ".join(finish_reasons))
|
| 102 |
+
|
| 103 |
+
openai_images: List[Image] = []
|
| 104 |
+
for _img in stability_3_response.get("images", []):
|
| 105 |
+
openai_images.append(Image(b64_json=_img))
|
| 106 |
+
|
| 107 |
+
model_response.data = openai_images
|
| 108 |
+
return model_response
|
| 109 |
+
|
| 110 |
+
@classmethod
|
| 111 |
+
def cost_calculator(
|
| 112 |
+
cls,
|
| 113 |
+
model: str,
|
| 114 |
+
image_response: ImageResponse,
|
| 115 |
+
size: Optional[str] = None,
|
| 116 |
+
optional_params: Optional[dict] = None,
|
| 117 |
+
) -> float:
|
| 118 |
+
get_model_info = get_cached_model_info()
|
| 119 |
+
model_info = get_model_info(
|
| 120 |
+
model=model,
|
| 121 |
+
custom_llm_provider="bedrock",
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
|
| 125 |
+
num_images: int = 0
|
| 126 |
+
if image_response.data:
|
| 127 |
+
num_images = len(image_response.data)
|
| 128 |
+
return output_cost_per_image * num_images
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic for Amazon Titan Image Generation.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import types
|
| 6 |
+
from typing import List, Optional
|
| 7 |
+
|
| 8 |
+
from openai.types.image import Image
|
| 9 |
+
|
| 10 |
+
from litellm.utils import get_model_info
|
| 11 |
+
from litellm.types.llms.bedrock import (
|
| 12 |
+
AmazonNovaCanvasImageGenerationConfig,
|
| 13 |
+
AmazonTitanImageGenerationRequestBody,
|
| 14 |
+
AmazonTitanTextToImageParams,
|
| 15 |
+
)
|
| 16 |
+
from litellm.types.utils import ImageResponse
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class AmazonTitanImageGenerationConfig:
|
| 20 |
+
"""
|
| 21 |
+
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=stability.stable-diffusion-xl-v0
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
cfg_scale: Optional[int] = None
|
| 25 |
+
seed: Optional[float] = None
|
| 26 |
+
steps: Optional[List[str]] = None
|
| 27 |
+
width: Optional[int] = None
|
| 28 |
+
height: Optional[int] = None
|
| 29 |
+
|
| 30 |
+
def __init__(
|
| 31 |
+
self,
|
| 32 |
+
cfg_scale: Optional[int] = None,
|
| 33 |
+
seed: Optional[float] = None,
|
| 34 |
+
steps: Optional[List[str]] = None,
|
| 35 |
+
width: Optional[int] = None,
|
| 36 |
+
height: Optional[int] = None,
|
| 37 |
+
) -> None:
|
| 38 |
+
locals_ = locals().copy()
|
| 39 |
+
for key, value in locals_.items():
|
| 40 |
+
if key != "self" and value is not None:
|
| 41 |
+
setattr(self.__class__, key, value)
|
| 42 |
+
|
| 43 |
+
@classmethod
|
| 44 |
+
def get_config(cls):
|
| 45 |
+
return {
|
| 46 |
+
k: v
|
| 47 |
+
for k, v in cls.__dict__.items()
|
| 48 |
+
if not k.startswith("__")
|
| 49 |
+
and not isinstance(
|
| 50 |
+
v,
|
| 51 |
+
(
|
| 52 |
+
types.FunctionType,
|
| 53 |
+
types.BuiltinFunctionType,
|
| 54 |
+
classmethod,
|
| 55 |
+
staticmethod,
|
| 56 |
+
),
|
| 57 |
+
)
|
| 58 |
+
and v is not None
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
@classmethod
|
| 62 |
+
def _is_titan_model(cls, model: Optional[str] = None) -> bool:
|
| 63 |
+
"""
|
| 64 |
+
Returns True if the model is a Titan model
|
| 65 |
+
|
| 66 |
+
Titan models follow this pattern:
|
| 67 |
+
|
| 68 |
+
"""
|
| 69 |
+
if model and "amazon.titan" in model:
|
| 70 |
+
return True
|
| 71 |
+
return False
|
| 72 |
+
|
| 73 |
+
@classmethod
|
| 74 |
+
def get_supported_openai_params(cls, model: Optional[str] = None) -> List:
|
| 75 |
+
return ["size", "n", "quality"]
|
| 76 |
+
|
| 77 |
+
@classmethod
|
| 78 |
+
def map_openai_params(
|
| 79 |
+
cls,
|
| 80 |
+
non_default_params: dict,
|
| 81 |
+
optional_params: dict,
|
| 82 |
+
):
|
| 83 |
+
from typing import Any, Dict
|
| 84 |
+
|
| 85 |
+
image_generation_config: Dict[str, Any] = {}
|
| 86 |
+
for k, v in non_default_params.items():
|
| 87 |
+
if k == "size" and v is not None:
|
| 88 |
+
width, height = v.split("x")
|
| 89 |
+
image_generation_config["width"] = int(width)
|
| 90 |
+
image_generation_config["height"] = int(height)
|
| 91 |
+
elif k == "n" and v is not None:
|
| 92 |
+
image_generation_config["numberOfImages"] = v
|
| 93 |
+
elif (
|
| 94 |
+
k == "quality" and v is not None
|
| 95 |
+
): # 'auto', 'hd', 'standard', 'high', 'medium', 'low'
|
| 96 |
+
if v in ("hd", "premium", "high"):
|
| 97 |
+
image_generation_config["quality"] = "premium"
|
| 98 |
+
elif v in ("standard", "medium", "low"):
|
| 99 |
+
image_generation_config["quality"] = "standard"
|
| 100 |
+
|
| 101 |
+
if image_generation_config:
|
| 102 |
+
optional_params["imageGenerationConfig"] = image_generation_config
|
| 103 |
+
return optional_params
|
| 104 |
+
|
| 105 |
+
@classmethod
|
| 106 |
+
def transform_request_body(
|
| 107 |
+
cls,
|
| 108 |
+
text: str,
|
| 109 |
+
optional_params: dict,
|
| 110 |
+
) -> AmazonTitanImageGenerationRequestBody:
|
| 111 |
+
from typing import Any, Dict
|
| 112 |
+
|
| 113 |
+
image_generation_config = optional_params.pop("imageGenerationConfig", {})
|
| 114 |
+
negative_text = optional_params.pop("negativeText", None)
|
| 115 |
+
text_to_image_params: Dict[str, Any] = {"text": text}
|
| 116 |
+
if negative_text:
|
| 117 |
+
text_to_image_params["negativeText"] = negative_text
|
| 118 |
+
task_type = optional_params.pop("taskType", "TEXT_IMAGE")
|
| 119 |
+
user_specified_image_generation_config = optional_params.pop(
|
| 120 |
+
"imageGenerationConfig", {}
|
| 121 |
+
)
|
| 122 |
+
image_generation_config = {
|
| 123 |
+
**image_generation_config,
|
| 124 |
+
**user_specified_image_generation_config,
|
| 125 |
+
}
|
| 126 |
+
return AmazonTitanImageGenerationRequestBody(
|
| 127 |
+
taskType=task_type,
|
| 128 |
+
textToImageParams=AmazonTitanTextToImageParams(**text_to_image_params), # type: ignore
|
| 129 |
+
imageGenerationConfig=AmazonNovaCanvasImageGenerationConfig(
|
| 130 |
+
**image_generation_config
|
| 131 |
+
),
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
@classmethod
|
| 135 |
+
def transform_response_dict_to_openai_response(
|
| 136 |
+
cls, model_response: ImageResponse, response_dict: dict
|
| 137 |
+
) -> ImageResponse:
|
| 138 |
+
image_list: List[Image] = []
|
| 139 |
+
for image in response_dict["images"]:
|
| 140 |
+
_image = Image(b64_json=image)
|
| 141 |
+
image_list.append(_image)
|
| 142 |
+
|
| 143 |
+
model_response.data = image_list
|
| 144 |
+
|
| 145 |
+
return model_response
|
| 146 |
+
|
| 147 |
+
@classmethod
|
| 148 |
+
def cost_calculator(
|
| 149 |
+
cls,
|
| 150 |
+
model: str,
|
| 151 |
+
image_response: ImageResponse,
|
| 152 |
+
size: Optional[str] = None,
|
| 153 |
+
optional_params: Optional[dict] = None,
|
| 154 |
+
) -> float:
|
| 155 |
+
model_info = get_model_info(model=model)
|
| 156 |
+
output_cost_per_image = model_info.get("output_cost_per_image") or 0.0
|
| 157 |
+
if not image_response.data:
|
| 158 |
+
return 0.0
|
| 159 |
+
num_images = len(image_response.data)
|
| 160 |
+
return output_cost_per_image * num_images
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/cost_calculator.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
from litellm.llms.bedrock.image_generation.image_handler import BedrockImageGeneration
|
| 4 |
+
from litellm.types.utils import ImageResponse
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def cost_calculator(
|
| 8 |
+
model: str,
|
| 9 |
+
image_response: ImageResponse,
|
| 10 |
+
size: Optional[str] = None,
|
| 11 |
+
optional_params: Optional[dict] = None,
|
| 12 |
+
) -> float:
|
| 13 |
+
"""
|
| 14 |
+
Bedrock image generation cost calculator
|
| 15 |
+
|
| 16 |
+
Handles both Stability 1 and Stability 3 models
|
| 17 |
+
"""
|
| 18 |
+
config_class = BedrockImageGeneration.get_config_class(model=model)
|
| 19 |
+
return config_class.cost_calculator(
|
| 20 |
+
model=model,
|
| 21 |
+
image_response=image_response,
|
| 22 |
+
size=size,
|
| 23 |
+
optional_params=optional_params,
|
| 24 |
+
)
|