techprotrade commited on
Commit
9fd948e
·
verified ·
1 Parent(s): 6541b2c

Add loan-advisor-agent (part 3)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/baseten/chat.py +121 -0
  2. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/base_aws_llm.py +1406 -0
  3. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/handler.py +97 -0
  4. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/transformation.py +549 -0
  5. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/__init__.py +30 -0
  6. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/__init__.py +3 -0
  7. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/transformation.py +1037 -0
  8. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_handler.py +512 -0
  9. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/handler.py +5 -0
  10. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/transformation.py +3 -0
  11. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_transformation.py +2126 -0
  12. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_agent/transformation.py +547 -0
  13. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_handler.py +1893 -0
  14. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py +99 -0
  15. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py +75 -0
  16. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +135 -0
  17. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py +80 -0
  18. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py +119 -0
  19. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +266 -0
  20. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +120 -0
  21. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py +192 -0
  22. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +99 -0
  23. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +225 -0
  24. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py +116 -0
  25. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +280 -0
  26. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py +98 -0
  27. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +206 -0
  28. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +613 -0
  29. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/common_utils.py +1140 -0
  30. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/cost_calculation.py +27 -0
  31. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +115 -0
  32. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/handler.py +143 -0
  33. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/transformation.py +272 -0
  34. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_nova_transformation.py +361 -0
  35. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py +88 -0
  36. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py +101 -0
  37. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py +131 -0
  38. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/cohere_transformation.py +47 -0
  39. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/embedding.py +699 -0
  40. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py +304 -0
  41. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/files/handler.py +210 -0
  42. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/files/transformation.py +786 -0
  43. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/__init__.py +9 -0
  44. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/handler.py +309 -0
  45. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/stability_transformation.py +399 -0
  46. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py +220 -0
  47. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py +164 -0
  48. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py +128 -0
  49. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py +160 -0
  50. loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/cost_calculator.py +24 -0
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/baseten/chat.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
3
+
4
+
5
+ class BasetenConfig(OpenAIGPTConfig):
6
+ """
7
+ Reference: https://inference.baseten.co/v1
8
+
9
+ Below are the parameters:
10
+ """
11
+
12
+ max_tokens: Optional[int] = None
13
+ response_format: Optional[dict] = None
14
+ seed: Optional[int] = None
15
+ stream: Optional[bool] = None
16
+ top_p: Optional[int] = None
17
+ tool_choice: Optional[str] = None
18
+ tools: Optional[list] = None
19
+ user: Optional[str] = None
20
+ presence_penalty: Optional[int] = None
21
+ frequency_penalty: Optional[int] = None
22
+ stream_options: Optional[dict] = None
23
+
24
+ def __init__(
25
+ self,
26
+ max_tokens: Optional[int] = None,
27
+ response_format: Optional[dict] = None,
28
+ seed: Optional[int] = None,
29
+ stop: Optional[list] = None,
30
+ stream: Optional[bool] = None,
31
+ temperature: Optional[float] = None,
32
+ top_p: Optional[int] = None,
33
+ tool_choice: Optional[str] = None,
34
+ tools: Optional[list] = None,
35
+ user: Optional[str] = None,
36
+ presence_penalty: Optional[int] = None,
37
+ frequency_penalty: Optional[int] = None,
38
+ stream_options: Optional[dict] = None,
39
+ ) -> None:
40
+ locals_ = locals().copy()
41
+ for key, value in locals_.items():
42
+ if key != "self" and value is not None:
43
+ setattr(self.__class__, key, value)
44
+
45
+ @classmethod
46
+ def get_config(cls):
47
+ return super().get_config()
48
+
49
+ def get_supported_openai_params(self, model: str) -> list:
50
+ """
51
+ Get the supported OpenAI params for the given model
52
+ """
53
+ return [
54
+ "max_tokens",
55
+ "max_completion_tokens",
56
+ "response_format",
57
+ "seed",
58
+ "stop",
59
+ "stream",
60
+ "temperature",
61
+ "top_p",
62
+ "tool_choice",
63
+ "tools",
64
+ "user",
65
+ "presence_penalty",
66
+ "frequency_penalty",
67
+ "stream_options",
68
+ ]
69
+
70
+ def map_openai_params(
71
+ self,
72
+ non_default_params: dict,
73
+ optional_params: dict,
74
+ model: str,
75
+ drop_params: bool,
76
+ ) -> dict:
77
+ supported_openai_params = self.get_supported_openai_params(model=model)
78
+ for param, value in non_default_params.items():
79
+ if param == "max_completion_tokens":
80
+ optional_params["max_tokens"] = value
81
+ elif param in supported_openai_params:
82
+ optional_params[param] = value
83
+ return optional_params
84
+
85
+ def _get_openai_compatible_provider_info(
86
+ self, api_base: str, api_key: str
87
+ ) -> tuple:
88
+ """
89
+ Get the OpenAI compatible provider info for Baseten
90
+ """
91
+ # Default to Model API
92
+ default_api_base = "https://inference.baseten.co/v1"
93
+ default_api_key = api_key or "BASETEN_API_KEY"
94
+
95
+ return default_api_base, default_api_key
96
+
97
+ @staticmethod
98
+ def is_dedicated_deployment(model: str) -> bool:
99
+ """
100
+ Check if the model is a dedicated deployment (8-digit alphanumeric code)
101
+ """
102
+ # Remove 'baseten/' prefix if present
103
+ model_id = model.replace("baseten/", "")
104
+
105
+ # Check if it's an 8-digit alphanumeric code
106
+ import re
107
+
108
+ return bool(re.match(r"^[a-zA-Z0-9]{8}$", model_id))
109
+
110
+ @staticmethod
111
+ def get_api_base_for_model(model: str) -> str:
112
+ """
113
+ Get the appropriate API base URL for the given model
114
+ """
115
+ if BasetenConfig.is_dedicated_deployment(model):
116
+ # Extract the model ID (remove 'baseten/' prefix if present)
117
+ model_id = model.replace("baseten/", "")
118
+ return f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1"
119
+ else:
120
+ # Use Model API
121
+ return "https://inference.baseten.co/v1"
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/base_aws_llm.py ADDED
@@ -0,0 +1,1406 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import hashlib
2
+ import json
3
+ import os
4
+ import urllib.parse
5
+ from datetime import datetime
6
+ from typing import (
7
+ TYPE_CHECKING,
8
+ Any,
9
+ Dict,
10
+ List,
11
+ Literal,
12
+ Optional,
13
+ Tuple,
14
+ Union,
15
+ cast,
16
+ get_args,
17
+ )
18
+
19
+ import httpx
20
+ from pydantic import BaseModel
21
+
22
+ from litellm._logging import verbose_logger
23
+ from litellm.caching.caching import DualCache
24
+ from litellm.constants import (
25
+ BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
26
+ BEDROCK_INVOKE_PROVIDERS_LITERAL,
27
+ BEDROCK_MAX_POLICY_SIZE,
28
+ )
29
+ from litellm.litellm_core_utils.dd_tracing import tracer
30
+ from litellm.secret_managers.main import get_secret, get_secret_str
31
+
32
+ if TYPE_CHECKING:
33
+ from botocore.awsrequest import AWSPreparedRequest
34
+ from botocore.credentials import Credentials
35
+ else:
36
+ Credentials = Any
37
+ AWSPreparedRequest = Any
38
+
39
+
40
+ class Boto3CredentialsInfo(BaseModel):
41
+ credentials: Credentials
42
+ aws_region_name: str
43
+ aws_bedrock_runtime_endpoint: Optional[str]
44
+
45
+
46
+ class AwsAuthError(Exception):
47
+ def __init__(self, status_code, message):
48
+ self.status_code = status_code
49
+ self.message = message
50
+ self.request = httpx.Request(
51
+ method="POST", url="https://us-west-2.console.aws.amazon.com/bedrock"
52
+ )
53
+ self.response = httpx.Response(status_code=status_code, request=self.request)
54
+ super().__init__(
55
+ self.message
56
+ ) # Call the base class constructor with the parameters it needs
57
+
58
+
59
+ class BaseAWSLLM:
60
+ def __init__(self) -> None:
61
+ self.iam_cache = DualCache()
62
+ super().__init__()
63
+ self.aws_authentication_params = [
64
+ "aws_access_key_id",
65
+ "aws_secret_access_key",
66
+ "aws_session_token",
67
+ "aws_region_name",
68
+ "aws_session_name",
69
+ "aws_profile_name",
70
+ "aws_role_name",
71
+ "aws_web_identity_token",
72
+ "aws_sts_endpoint",
73
+ "aws_bedrock_runtime_endpoint",
74
+ "aws_external_id",
75
+ ]
76
+
77
+ def _get_ssl_verify(self, ssl_verify: Optional[Union[bool, str]] = None):
78
+ """
79
+ Get SSL verification setting for boto3 clients.
80
+
81
+ This ensures that custom CA certificates are properly used for all AWS API calls,
82
+ including STS and Bedrock services.
83
+
84
+ Returns:
85
+ Union[bool, str]: SSL verification setting - False to disable, True to enable,
86
+ or a string path to a CA bundle file
87
+ """
88
+ from litellm.llms.custom_httpx.http_handler import get_ssl_verify
89
+
90
+ return get_ssl_verify(ssl_verify=ssl_verify)
91
+
92
+ def get_cache_key(self, credential_args: Dict[str, Optional[str]]) -> str:
93
+ """
94
+ Generate a unique cache key based on the credential arguments.
95
+ """
96
+ # Convert credential arguments to a JSON string and hash it to create a unique key
97
+ credential_str = json.dumps(credential_args, sort_keys=True)
98
+ return hashlib.sha256(credential_str.encode()).hexdigest()
99
+
100
+ @tracer.wrap()
101
+ def get_credentials(
102
+ self,
103
+ aws_access_key_id: Optional[str] = None,
104
+ aws_secret_access_key: Optional[str] = None,
105
+ aws_session_token: Optional[str] = None,
106
+ aws_region_name: Optional[str] = None,
107
+ aws_session_name: Optional[str] = None,
108
+ aws_profile_name: Optional[str] = None,
109
+ aws_role_name: Optional[str] = None,
110
+ aws_web_identity_token: Optional[str] = None,
111
+ aws_sts_endpoint: Optional[str] = None,
112
+ aws_external_id: Optional[str] = None,
113
+ ssl_verify: Optional[Union[bool, str]] = None,
114
+ ):
115
+ """
116
+ Return a boto3.Credentials object
117
+ """
118
+ ## CHECK IS 'os.environ/' passed in
119
+ params_to_check: List[Optional[str]] = [
120
+ aws_access_key_id,
121
+ aws_secret_access_key,
122
+ aws_session_token,
123
+ aws_region_name,
124
+ aws_session_name,
125
+ aws_profile_name,
126
+ aws_role_name,
127
+ aws_web_identity_token,
128
+ aws_sts_endpoint,
129
+ aws_external_id,
130
+ ]
131
+
132
+ # Iterate over parameters and update if needed
133
+ for i, param in enumerate(params_to_check):
134
+ if param and param.startswith("os.environ/"):
135
+ _v = get_secret(param)
136
+ if _v is not None and isinstance(_v, str):
137
+ params_to_check[i] = _v
138
+ elif param is None: # check if uppercase value in env
139
+ key = self.aws_authentication_params[i]
140
+ if key.upper() in os.environ:
141
+ params_to_check[i] = os.getenv(key.upper())
142
+
143
+ # Assign updated values back to parameters
144
+ (
145
+ aws_access_key_id,
146
+ aws_secret_access_key,
147
+ aws_session_token,
148
+ aws_region_name,
149
+ aws_session_name,
150
+ aws_profile_name,
151
+ aws_role_name,
152
+ aws_web_identity_token,
153
+ aws_sts_endpoint,
154
+ aws_external_id,
155
+ ) = params_to_check
156
+
157
+ verbose_logger.debug(
158
+ "in get credentials\n"
159
+ "aws_access_key_id=%s\n"
160
+ "aws_secret_access_key=%s\n"
161
+ "aws_session_token=%s\n"
162
+ "aws_region_name=%s\n"
163
+ "aws_session_name=%s\n"
164
+ "aws_profile_name=%s\n"
165
+ "aws_role_name=%s\n"
166
+ "aws_web_identity_token=%s\n"
167
+ "aws_sts_endpoint=%s\n"
168
+ "aws_external_id=%s",
169
+ aws_access_key_id,
170
+ aws_secret_access_key,
171
+ aws_session_token,
172
+ aws_region_name,
173
+ aws_session_name,
174
+ aws_profile_name,
175
+ aws_role_name,
176
+ aws_web_identity_token,
177
+ aws_sts_endpoint,
178
+ aws_external_id,
179
+ )
180
+
181
+ # create cache key for non-expiring auth flows
182
+ args = {
183
+ k: v
184
+ for k, v in locals().items()
185
+ if k.startswith("aws_") or k == "ssl_verify"
186
+ }
187
+
188
+ cache_key = self.get_cache_key(args)
189
+ _cached_credentials = self.iam_cache.get_cache(cache_key)
190
+ if _cached_credentials:
191
+ return _cached_credentials
192
+
193
+ #########################################################
194
+ # Handle diff boto3 auth flows
195
+ # for each helper
196
+ # Return:
197
+ # Credentials - boto3.Credentials
198
+ # cache ttl - Optional[int]. If None, the credentials are not cached. Some auth flows have no expiry time.
199
+ #########################################################
200
+ if (
201
+ aws_web_identity_token is not None
202
+ and aws_role_name is not None
203
+ and aws_session_name is not None
204
+ ):
205
+ credentials, _cache_ttl = self._auth_with_web_identity_token(
206
+ aws_web_identity_token=aws_web_identity_token,
207
+ aws_role_name=aws_role_name,
208
+ aws_session_name=aws_session_name,
209
+ aws_region_name=aws_region_name,
210
+ aws_sts_endpoint=aws_sts_endpoint,
211
+ aws_external_id=aws_external_id,
212
+ )
213
+ elif aws_role_name is not None:
214
+ # Check if we're already running as the target role and can skip assumption
215
+ # This handles IRSA (EKS), ECS task roles, and EC2 instance profiles
216
+ if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
217
+ verbose_logger.debug(
218
+ "Already running as target role %s, using ambient credentials",
219
+ aws_role_name,
220
+ )
221
+ credentials, _cache_ttl = self._auth_with_env_vars()
222
+ else:
223
+ verbose_logger.debug(
224
+ "Using role assumption: calling _auth_with_aws_role"
225
+ )
226
+ # If aws_session_name is not provided, generate a default one
227
+ if aws_session_name is None:
228
+ aws_session_name = (
229
+ f"litellm-session-{int(datetime.now().timestamp())}"
230
+ )
231
+ credentials, _cache_ttl = self._auth_with_aws_role(
232
+ aws_access_key_id=aws_access_key_id,
233
+ aws_secret_access_key=aws_secret_access_key,
234
+ aws_session_token=aws_session_token,
235
+ aws_role_name=aws_role_name,
236
+ aws_session_name=aws_session_name,
237
+ aws_region_name=aws_region_name,
238
+ aws_sts_endpoint=aws_sts_endpoint,
239
+ aws_external_id=aws_external_id,
240
+ ssl_verify=ssl_verify,
241
+ )
242
+
243
+ elif aws_profile_name is not None: ### CHECK SESSION ###
244
+ credentials, _cache_ttl = self._auth_with_aws_profile(aws_profile_name)
245
+ elif (
246
+ aws_access_key_id is not None
247
+ and aws_secret_access_key is not None
248
+ and aws_session_token is not None
249
+ ):
250
+ credentials, _cache_ttl = self._auth_with_aws_session_token(
251
+ aws_access_key_id=aws_access_key_id,
252
+ aws_secret_access_key=aws_secret_access_key,
253
+ aws_session_token=aws_session_token,
254
+ )
255
+ elif (
256
+ aws_access_key_id is not None
257
+ and aws_secret_access_key is not None
258
+ and aws_region_name is not None
259
+ ):
260
+ credentials, _cache_ttl = self._auth_with_access_key_and_secret_key(
261
+ aws_access_key_id=aws_access_key_id,
262
+ aws_secret_access_key=aws_secret_access_key,
263
+ aws_region_name=aws_region_name,
264
+ )
265
+ else:
266
+ credentials, _cache_ttl = self._auth_with_env_vars()
267
+
268
+ self.iam_cache.set_cache(cache_key, credentials, ttl=_cache_ttl)
269
+ return credentials
270
+
271
+ def _get_aws_region_from_model_arn(self, model: Optional[str]) -> Optional[str]:
272
+ try:
273
+ # First check if the string contains the expected prefix
274
+ if not isinstance(model, str) or "arn:aws:bedrock" not in model:
275
+ return None
276
+
277
+ # Split the ARN and check if we have enough parts
278
+ parts = model.split(":")
279
+ if len(parts) < 4:
280
+ return None
281
+
282
+ # Get the region from the correct position
283
+ region = parts[3]
284
+ if not region: # Check if region is empty
285
+ return None
286
+
287
+ return region
288
+ except Exception:
289
+ # Catch any unexpected errors and return None
290
+ return None
291
+
292
+ @staticmethod
293
+ def _get_provider_from_model_path(
294
+ model_path: str,
295
+ ) -> Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL]:
296
+ """
297
+ Helper function to get the provider from a model path with format: provider/model-name
298
+
299
+ Args:
300
+ model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
301
+
302
+ Returns:
303
+ Optional[str]: The provider name, or None if no valid provider found
304
+ """
305
+ parts = model_path.split("/")
306
+ if len(parts) >= 1:
307
+ provider = parts[0]
308
+ if provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
309
+ return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
310
+ return None
311
+
312
+ @staticmethod
313
+ def get_bedrock_invoke_provider(
314
+ model: str,
315
+ ) -> Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL]:
316
+ """
317
+ Helper function to get the bedrock provider from the model
318
+
319
+ handles 3 scenarions:
320
+ 1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
321
+ 2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
322
+ 3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
323
+ 4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
324
+ """
325
+ if model.startswith("invoke/"):
326
+ model = model.replace("invoke/", "", 1)
327
+
328
+ # Special case: Check for "nova" in model name first (before "amazon")
329
+ # This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
330
+ if "nova" in model.lower():
331
+ if "nova" in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
332
+ return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
333
+
334
+ _split_model = model.split(".")[0]
335
+ if _split_model in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
336
+ return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
337
+
338
+ # If not a known provider, check for pattern with two slashes
339
+ provider = BaseAWSLLM._get_provider_from_model_path(model)
340
+ if provider is not None:
341
+ return provider
342
+
343
+ for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
344
+ if provider in model:
345
+ return provider
346
+ return None
347
+
348
+ @staticmethod
349
+ def get_bedrock_model_id(
350
+ optional_params: dict,
351
+ provider: Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL],
352
+ model: str,
353
+ ) -> str:
354
+ model_id = optional_params.pop("model_id", None)
355
+ if model_id is not None:
356
+ model_id = BaseAWSLLM.encode_model_id(model_id=model_id)
357
+ else:
358
+ model_id = model
359
+
360
+ model_id = model_id.replace("invoke/", "", 1)
361
+ if provider == "llama" and "llama/" in model_id:
362
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
363
+ model_id, spec="llama"
364
+ )
365
+ elif provider == "deepseek_r1" and "deepseek_r1/" in model_id:
366
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
367
+ model_id, spec="deepseek_r1"
368
+ )
369
+ elif provider == "openai" and "openai/" in model_id:
370
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
371
+ model_id, spec="openai"
372
+ )
373
+ elif provider == "qwen2" and "qwen2/" in model_id:
374
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
375
+ model_id, spec="qwen2"
376
+ )
377
+ elif provider == "qwen3" and "qwen3/" in model_id:
378
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
379
+ model_id, spec="qwen3"
380
+ )
381
+ elif provider == "stability" and "stability/" in model_id:
382
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
383
+ model_id, spec="stability"
384
+ )
385
+ elif provider == "moonshot" and "moonshot/" in model_id:
386
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
387
+ model_id, spec="moonshot"
388
+ )
389
+ elif "nova-2/" in model_id:
390
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
391
+ model_id, spec="nova-2"
392
+ )
393
+ elif "nova/" in model_id:
394
+ model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
395
+ model_id, spec="nova"
396
+ )
397
+ return model_id
398
+
399
+ @staticmethod
400
+ def _get_model_id_from_model_with_spec(
401
+ model: str,
402
+ spec: str,
403
+ ) -> str:
404
+ """
405
+ Remove `llama` from modelID since `llama` is simply a spec to follow for custom bedrock models
406
+ """
407
+ model_id = model.replace(spec + "/", "")
408
+ return BaseAWSLLM.encode_model_id(model_id=model_id)
409
+
410
+ @staticmethod
411
+ def encode_model_id(model_id: str) -> str:
412
+ """
413
+ Double encode the model ID to ensure it matches the expected double-encoded format.
414
+ Args:
415
+ model_id (str): The model ID to encode.
416
+ Returns:
417
+ str: The double-encoded model ID.
418
+ """
419
+ return urllib.parse.quote(model_id, safe="")
420
+
421
+ @staticmethod
422
+ def get_bedrock_embedding_provider(
423
+ model: str,
424
+ ) -> Optional[BEDROCK_EMBEDDING_PROVIDERS_LITERAL]:
425
+ """
426
+ Helper function to get the bedrock embedding provider from the model
427
+
428
+ Handles scenarios like:
429
+ 1. model=cohere.embed-english-v3:0 -> Returns `cohere`
430
+ 2. model=amazon.titan-embed-text-v1 -> Returns `amazon`
431
+ 3. model=amazon.nova-2-multimodal-embeddings-v1:0 -> Returns `nova`
432
+ 4. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
433
+ 5. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
434
+ """
435
+ # Special case: Check for "nova" in model name first (before "amazon")
436
+ # This handles amazon.nova-* models
437
+ if "nova" in model.lower():
438
+ if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
439
+ return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova")
440
+
441
+ # Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0
442
+ if "." in model:
443
+ parts = model.split(".")
444
+ # Check if the second part (after potential region) is a known provider
445
+ if len(parts) >= 2:
446
+ potential_provider = parts[
447
+ 1
448
+ ] # e.g., "twelvelabs" from "us.twelvelabs.marengo-embed-2-7-v1:0"
449
+ if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
450
+ return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider)
451
+
452
+ # Check if the first part is a known provider (standard format)
453
+ potential_provider = parts[
454
+ 0
455
+ ] # e.g., "cohere" from "cohere.embed-english-v3:0"
456
+ if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
457
+ return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider)
458
+
459
+ # Fallback: check if any provider name appears in the model string
460
+ for provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
461
+ if provider in model:
462
+ return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, provider)
463
+
464
+ return None
465
+
466
+ def _get_aws_region_name(
467
+ self,
468
+ optional_params: dict,
469
+ model: Optional[str] = None,
470
+ model_id: Optional[str] = None,
471
+ ) -> str:
472
+ """
473
+ Get the AWS region name from the environment variables.
474
+
475
+ Parameters:
476
+ optional_params (dict): Optional parameters for the model call
477
+ model (str): The model name
478
+ model_id (str): The model ID. This is the ARN of the model, if passed in as a separate param.
479
+
480
+ Returns:
481
+ str: The AWS region name
482
+ """
483
+ aws_region_name = optional_params.get("aws_region_name", None)
484
+ ### SET REGION NAME ###
485
+ if aws_region_name is None:
486
+ # check model arn #
487
+ if model_id is not None:
488
+ aws_region_name = self._get_aws_region_from_model_arn(model_id)
489
+ else:
490
+ aws_region_name = self._get_aws_region_from_model_arn(model)
491
+ # check env #
492
+ litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
493
+
494
+ if (
495
+ aws_region_name is None
496
+ and litellm_aws_region_name is not None
497
+ and isinstance(litellm_aws_region_name, str)
498
+ ):
499
+ aws_region_name = litellm_aws_region_name
500
+
501
+ standard_aws_region_name = get_secret("AWS_REGION", None)
502
+ if (
503
+ aws_region_name is None
504
+ and standard_aws_region_name is not None
505
+ and isinstance(standard_aws_region_name, str)
506
+ ):
507
+ aws_region_name = standard_aws_region_name
508
+ if aws_region_name is None:
509
+ try:
510
+ import boto3
511
+
512
+ with tracer.trace("boto3.Session()"):
513
+ session = boto3.Session()
514
+ configured_region = session.region_name
515
+ if configured_region:
516
+ aws_region_name = configured_region
517
+ else:
518
+ aws_region_name = "us-west-2"
519
+ except Exception:
520
+ aws_region_name = "us-west-2"
521
+
522
+ return aws_region_name
523
+
524
+ def get_aws_region_name_for_non_llm_api_calls(
525
+ self,
526
+ aws_region_name: Optional[str] = None,
527
+ ):
528
+ """
529
+ Get the AWS region name for non-llm api calls.
530
+
531
+ LLM API calls check the model arn and end up using that as the region name.
532
+
533
+ For non-llm api calls eg. Guardrails, Vector Stores we just need to check the dynamic param or env vars.
534
+ """
535
+ if aws_region_name is None:
536
+ # check env #
537
+ litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
538
+
539
+ if litellm_aws_region_name is not None and isinstance(
540
+ litellm_aws_region_name, str
541
+ ):
542
+ aws_region_name = litellm_aws_region_name
543
+
544
+ standard_aws_region_name = get_secret("AWS_REGION", None)
545
+ if standard_aws_region_name is not None and isinstance(
546
+ standard_aws_region_name, str
547
+ ):
548
+ aws_region_name = standard_aws_region_name
549
+
550
+ if aws_region_name is None:
551
+ aws_region_name = "us-west-2"
552
+ return aws_region_name
553
+
554
+ @staticmethod
555
+ def _parse_arn_account_and_role_name(
556
+ arn: str,
557
+ ) -> Optional[Tuple[str, str, str]]:
558
+ """
559
+ Parse an ARN and return (partition, account_id, role_name).
560
+
561
+ Handles:
562
+ - arn:aws:iam::123456789012:role/MyRole
563
+ - arn:aws:iam::123456789012:role/path/to/MyRole
564
+ - arn:aws:sts::123456789012:assumed-role/MyRole/session-name
565
+
566
+ Returns None if the ARN cannot be parsed.
567
+ """
568
+ # ARN format: arn:PARTITION:SERVICE:REGION:ACCOUNT:RESOURCE
569
+ parts = arn.split(":")
570
+ if len(parts) < 6 or parts[0] != "arn":
571
+ return None
572
+
573
+ partition = parts[1] # e.g. "aws", "aws-cn", "aws-us-gov"
574
+ account_id = parts[4]
575
+ resource = ":".join(parts[5:]) # rejoin in case resource contains colons
576
+
577
+ if resource.startswith("role/"):
578
+ # arn:aws:iam::ACCOUNT:role/[path/]ROLE_NAME
579
+ role_name = resource.split("/")[-1]
580
+ elif resource.startswith("assumed-role/"):
581
+ # arn:aws:sts::ACCOUNT:assumed-role/ROLE_NAME/SESSION
582
+ role_parts = resource.split("/")
583
+ if len(role_parts) >= 2:
584
+ role_name = role_parts[1]
585
+ else:
586
+ return None
587
+ else:
588
+ return None
589
+
590
+ return partition, account_id, role_name
591
+
592
+ def _is_already_running_as_role(
593
+ self,
594
+ aws_role_name: str,
595
+ ssl_verify: Optional[Union[bool, str]] = None,
596
+ ) -> bool:
597
+ """
598
+ Check if the current environment is already running as the target IAM role.
599
+
600
+ This handles multiple AWS environments:
601
+ - IRSA (EKS): AWS_ROLE_ARN + AWS_WEB_IDENTITY_TOKEN_FILE are set
602
+ - ECS task roles: Uses sts:GetCallerIdentity to check current role ARN
603
+ - EC2 instance profiles: Uses sts:GetCallerIdentity to check current role ARN
604
+
605
+ Compares partition, account ID, and role name to avoid cross-account
606
+ false matches.
607
+
608
+ Returns True if the current identity matches the target role, meaning
609
+ we can skip sts:AssumeRole and use ambient credentials directly.
610
+ """
611
+ target_parsed = self._parse_arn_account_and_role_name(aws_role_name)
612
+ if target_parsed is None:
613
+ return False
614
+
615
+ target_partition, target_account, target_role = target_parsed
616
+
617
+ # Fast path: IRSA environment check (no API call needed)
618
+ current_role_arn = os.getenv("AWS_ROLE_ARN")
619
+ web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
620
+ if current_role_arn and web_identity_token_file:
621
+ return current_role_arn == aws_role_name
622
+
623
+ # For ECS/EC2: call sts:GetCallerIdentity to check if already running as the role
624
+ try:
625
+ import boto3
626
+
627
+ with tracer.trace("boto3.client(sts).get_caller_identity"):
628
+ sts_client = boto3.client(
629
+ "sts", verify=self._get_ssl_verify(ssl_verify)
630
+ )
631
+ identity = sts_client.get_caller_identity()
632
+ caller_arn = identity.get("Arn", "")
633
+
634
+ caller_parsed = self._parse_arn_account_and_role_name(caller_arn)
635
+ if caller_parsed is not None:
636
+ caller_partition, caller_account, caller_role = caller_parsed
637
+ if (
638
+ caller_partition == target_partition
639
+ and caller_account == target_account
640
+ and caller_role == target_role
641
+ ):
642
+ verbose_logger.debug(
643
+ "Current identity already matches target role: %s",
644
+ aws_role_name,
645
+ )
646
+ return True
647
+
648
+ except Exception as e:
649
+ verbose_logger.debug(
650
+ "Could not determine current role identity: %s", str(e)
651
+ )
652
+
653
+ return False
654
+
655
+ @tracer.wrap()
656
+ def _auth_with_web_identity_token(
657
+ self,
658
+ aws_web_identity_token: str,
659
+ aws_role_name: str,
660
+ aws_session_name: str,
661
+ aws_region_name: Optional[str],
662
+ aws_sts_endpoint: Optional[str],
663
+ aws_external_id: Optional[str] = None,
664
+ ssl_verify: Optional[Union[bool, str]] = None,
665
+ ) -> Tuple[Credentials, Optional[int]]:
666
+ """
667
+ Authenticate with AWS Web Identity Token
668
+ """
669
+ import boto3
670
+
671
+ verbose_logger.debug(
672
+ f"IN Web Identity Token: {aws_web_identity_token} | Role Name: {aws_role_name} | Session Name: {aws_session_name}"
673
+ )
674
+
675
+ if aws_sts_endpoint is None:
676
+ sts_endpoint = f"https://sts.{aws_region_name}.amazonaws.com"
677
+ else:
678
+ sts_endpoint = aws_sts_endpoint
679
+
680
+ oidc_token = get_secret(aws_web_identity_token)
681
+
682
+ if oidc_token is None:
683
+ raise AwsAuthError(
684
+ message="OIDC token could not be retrieved from secret manager.",
685
+ status_code=401,
686
+ )
687
+
688
+ with tracer.trace("boto3.client(sts)"):
689
+ sts_client = boto3.client(
690
+ "sts",
691
+ region_name=aws_region_name,
692
+ endpoint_url=sts_endpoint,
693
+ verify=self._get_ssl_verify(ssl_verify),
694
+ )
695
+
696
+ # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
697
+ # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
698
+ assume_role_params = {
699
+ "RoleArn": aws_role_name,
700
+ "RoleSessionName": aws_session_name,
701
+ "WebIdentityToken": oidc_token,
702
+ "DurationSeconds": 3600,
703
+ "Policy": '{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringLike":{"aws:UserAgent":"litellm/*"}}}]}',
704
+ }
705
+
706
+ # Add ExternalId parameter if provided
707
+ if aws_external_id is not None:
708
+ assume_role_params["ExternalId"] = aws_external_id
709
+
710
+ sts_response = sts_client.assume_role_with_web_identity(**assume_role_params)
711
+
712
+ iam_creds_dict = {
713
+ "aws_access_key_id": sts_response["Credentials"]["AccessKeyId"],
714
+ "aws_secret_access_key": sts_response["Credentials"]["SecretAccessKey"],
715
+ "aws_session_token": sts_response["Credentials"]["SessionToken"],
716
+ "region_name": aws_region_name,
717
+ }
718
+
719
+ if sts_response["PackedPolicySize"] > BEDROCK_MAX_POLICY_SIZE:
720
+ verbose_logger.warning(
721
+ f"The policy size is greater than 75% of the allowed size, PackedPolicySize: {sts_response['PackedPolicySize']}"
722
+ )
723
+
724
+ with tracer.trace("boto3.Session(**iam_creds_dict)"):
725
+ session = boto3.Session(**iam_creds_dict)
726
+
727
+ iam_creds = session.get_credentials()
728
+ return iam_creds, self._get_default_ttl_for_boto3_credentials()
729
+
730
+ def _handle_irsa_cross_account(
731
+ self,
732
+ irsa_role_arn: str,
733
+ aws_role_name: str,
734
+ aws_session_name: str,
735
+ region: str,
736
+ web_identity_token_file: str,
737
+ aws_external_id: Optional[str] = None,
738
+ aws_sts_endpoint: Optional[str] = None,
739
+ ssl_verify: Optional[Union[bool, str]] = None,
740
+ ) -> dict:
741
+ """Handle cross-account role assumption for IRSA."""
742
+ import boto3
743
+
744
+ verbose_logger.debug("Cross-account role assumption detected")
745
+
746
+ # Read the web identity token
747
+ with open(web_identity_token_file, "r") as f:
748
+ web_identity_token = f.read().strip()
749
+
750
+ irsa_sts_kwargs: dict = {
751
+ "region_name": region,
752
+ "verify": self._get_ssl_verify(ssl_verify),
753
+ }
754
+ if aws_sts_endpoint is not None:
755
+ irsa_sts_kwargs["endpoint_url"] = aws_sts_endpoint
756
+
757
+ # Create an STS client without credentials
758
+ with tracer.trace("boto3.client(sts) for manual IRSA"):
759
+ sts_client = boto3.client("sts", **irsa_sts_kwargs)
760
+
761
+ # Manually assume the IRSA role with the session name
762
+ verbose_logger.debug(
763
+ f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}"
764
+ )
765
+ irsa_response = sts_client.assume_role_with_web_identity(
766
+ RoleArn=irsa_role_arn,
767
+ RoleSessionName=aws_session_name,
768
+ WebIdentityToken=web_identity_token,
769
+ )
770
+
771
+ # Extract the credentials from the IRSA assumption
772
+ irsa_creds = irsa_response["Credentials"]
773
+
774
+ # Create a new STS client with the IRSA credentials
775
+ with tracer.trace("boto3.client(sts) with manual IRSA credentials"):
776
+ sts_client_with_creds = boto3.client(
777
+ "sts",
778
+ aws_access_key_id=irsa_creds["AccessKeyId"],
779
+ aws_secret_access_key=irsa_creds["SecretAccessKey"],
780
+ aws_session_token=irsa_creds["SessionToken"],
781
+ **irsa_sts_kwargs,
782
+ )
783
+
784
+ # Get current caller identity for debugging
785
+ try:
786
+ caller_identity = sts_client_with_creds.get_caller_identity()
787
+ verbose_logger.debug(
788
+ f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}"
789
+ )
790
+ except Exception as e:
791
+ verbose_logger.debug(f"Failed to get caller identity: {e}")
792
+
793
+ # Now assume the target role
794
+ verbose_logger.debug(
795
+ f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}"
796
+ )
797
+ assume_role_params = {
798
+ "RoleArn": aws_role_name,
799
+ "RoleSessionName": aws_session_name,
800
+ }
801
+
802
+ # Add ExternalId parameter if provided
803
+ if aws_external_id is not None:
804
+ assume_role_params["ExternalId"] = aws_external_id
805
+
806
+ return sts_client_with_creds.assume_role(**assume_role_params)
807
+
808
+ def _handle_irsa_same_account(
809
+ self,
810
+ aws_role_name: str,
811
+ aws_session_name: str,
812
+ region: str,
813
+ aws_external_id: Optional[str] = None,
814
+ aws_sts_endpoint: Optional[str] = None,
815
+ ssl_verify: Optional[Union[bool, str]] = None,
816
+ ) -> dict:
817
+ """Handle same-account role assumption for IRSA."""
818
+ import boto3
819
+
820
+ irsa_sts_kwargs: dict = {
821
+ "region_name": region,
822
+ "verify": self._get_ssl_verify(ssl_verify),
823
+ }
824
+ if aws_sts_endpoint is not None:
825
+ irsa_sts_kwargs["endpoint_url"] = aws_sts_endpoint
826
+
827
+ verbose_logger.debug("Same account role assumption, using automatic IRSA")
828
+ with tracer.trace("boto3.client(sts) with automatic IRSA"):
829
+ sts_client = boto3.client("sts", **irsa_sts_kwargs)
830
+
831
+ # Get current caller identity for debugging
832
+ try:
833
+ caller_identity = sts_client.get_caller_identity()
834
+ verbose_logger.debug(
835
+ f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}"
836
+ )
837
+ except Exception as e:
838
+ verbose_logger.debug(f"Failed to get caller identity: {e}")
839
+
840
+ # Assume the role
841
+ verbose_logger.debug(
842
+ f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}"
843
+ )
844
+ assume_role_params = {
845
+ "RoleArn": aws_role_name,
846
+ "RoleSessionName": aws_session_name,
847
+ }
848
+
849
+ # Add ExternalId parameter if provided
850
+ if aws_external_id is not None:
851
+ assume_role_params["ExternalId"] = aws_external_id
852
+
853
+ return sts_client.assume_role(**assume_role_params)
854
+
855
+ def _extract_credentials_and_ttl(
856
+ self, sts_response: dict
857
+ ) -> Tuple[Credentials, Optional[int]]:
858
+ """Extract credentials and TTL from STS response."""
859
+ from botocore.credentials import Credentials
860
+
861
+ sts_credentials = sts_response["Credentials"]
862
+ credentials = Credentials(
863
+ access_key=sts_credentials["AccessKeyId"],
864
+ secret_key=sts_credentials["SecretAccessKey"],
865
+ token=sts_credentials["SessionToken"],
866
+ )
867
+
868
+ expiration_time = sts_credentials["Expiration"]
869
+ ttl = int(
870
+ (expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds()
871
+ )
872
+
873
+ return credentials, ttl
874
+
875
+ @tracer.wrap()
876
+ def _auth_with_aws_role(
877
+ self,
878
+ aws_access_key_id: Optional[str],
879
+ aws_secret_access_key: Optional[str],
880
+ aws_session_token: Optional[str],
881
+ aws_role_name: str,
882
+ aws_session_name: str,
883
+ aws_region_name: Optional[str] = None,
884
+ aws_sts_endpoint: Optional[str] = None,
885
+ aws_external_id: Optional[str] = None,
886
+ ssl_verify: Optional[Union[bool, str]] = None,
887
+ ) -> Tuple[Credentials, Optional[int]]:
888
+ """
889
+ Authenticate with AWS Role
890
+ """
891
+ import boto3
892
+ from botocore.credentials import Credentials
893
+
894
+ # Check if we're in an EKS/IRSA environment
895
+ web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
896
+ irsa_role_arn = os.getenv("AWS_ROLE_ARN")
897
+
898
+ region = (
899
+ aws_region_name
900
+ or os.getenv("AWS_REGION")
901
+ or os.getenv("AWS_DEFAULT_REGION")
902
+ )
903
+
904
+ # If we have IRSA environment variables and no explicit credentials,
905
+ # we need to use the web identity token flow
906
+ if (
907
+ web_identity_token_file
908
+ and irsa_role_arn
909
+ and aws_access_key_id is None
910
+ and aws_secret_access_key is None
911
+ ):
912
+ # For cross-account role assumption with specific session names,
913
+ # we need to manually assume the IRSA role first with the correct session name
914
+ verbose_logger.debug(
915
+ f"IRSA detected: using web identity token from {web_identity_token_file}"
916
+ )
917
+
918
+ try:
919
+ # Use passed-in region when set, else env, else default (align with AssumeRole path)
920
+ region = region or "us-east-1"
921
+
922
+ # Check if we need to do cross-account role assumption
923
+ if aws_role_name != irsa_role_arn:
924
+ sts_response = self._handle_irsa_cross_account(
925
+ irsa_role_arn,
926
+ aws_role_name,
927
+ aws_session_name,
928
+ region,
929
+ web_identity_token_file,
930
+ aws_external_id,
931
+ aws_sts_endpoint=aws_sts_endpoint,
932
+ ssl_verify=ssl_verify,
933
+ )
934
+ else:
935
+ sts_response = self._handle_irsa_same_account(
936
+ aws_role_name,
937
+ aws_session_name,
938
+ region,
939
+ aws_external_id,
940
+ aws_sts_endpoint=aws_sts_endpoint,
941
+ ssl_verify=ssl_verify,
942
+ )
943
+
944
+ return self._extract_credentials_and_ttl(sts_response)
945
+
946
+ except Exception as e:
947
+ verbose_logger.debug(f"Failed to assume role via IRSA: {e}")
948
+ if "AccessDenied" in str(
949
+ e
950
+ ) and "is not authorized to perform: sts:AssumeRole" in str(e):
951
+ # Provide a more helpful error message for trust policy issues
952
+ verbose_logger.error(
953
+ f"Access denied when trying to assume role {aws_role_name}. "
954
+ f"Please ensure the trust policy of {aws_role_name} allows "
955
+ f"the current role to assume it. Current identity: check logs with verbose mode."
956
+ )
957
+ # Re-raise the exception instead of falling through
958
+ raise
959
+
960
+ # In EKS/IRSA environments, use ambient credentials (no explicit keys needed)
961
+ # This allows the web identity token to work automatically
962
+ sts_client_kwargs: dict = {"verify": self._get_ssl_verify(ssl_verify)}
963
+ if region is not None:
964
+ sts_client_kwargs["region_name"] = region
965
+ if aws_sts_endpoint is not None:
966
+ sts_client_kwargs["endpoint_url"] = aws_sts_endpoint
967
+ if aws_access_key_id is None and aws_secret_access_key is None:
968
+ with tracer.trace("boto3.client(sts)"):
969
+ sts_client = boto3.client("sts", **sts_client_kwargs)
970
+ else:
971
+ with tracer.trace("boto3.client(sts)"):
972
+ sts_client = boto3.client(
973
+ "sts",
974
+ aws_access_key_id=aws_access_key_id,
975
+ aws_secret_access_key=aws_secret_access_key,
976
+ aws_session_token=aws_session_token,
977
+ **sts_client_kwargs,
978
+ )
979
+
980
+ assume_role_params = {
981
+ "RoleArn": aws_role_name,
982
+ "RoleSessionName": aws_session_name,
983
+ }
984
+
985
+ # Add ExternalId parameter if provided
986
+ if aws_external_id is not None:
987
+ assume_role_params["ExternalId"] = aws_external_id
988
+
989
+ try:
990
+ sts_response = sts_client.assume_role(**assume_role_params)
991
+ except Exception as e:
992
+ error_str = str(e)
993
+ if "AccessDenied" in error_str:
994
+ # Only fall back to ambient credentials if we can positively
995
+ # confirm the caller is already the target role (same account,
996
+ # partition, and role name). This avoids silently using the
997
+ # wrong identity when there is a genuine trust-policy or
998
+ # permission misconfiguration.
999
+ if self._is_already_running_as_role(
1000
+ aws_role_name, ssl_verify=ssl_verify
1001
+ ):
1002
+ verbose_logger.warning(
1003
+ "AssumeRole failed for %s (%s). "
1004
+ "Caller is already running as this role; "
1005
+ "falling back to ambient credentials.",
1006
+ aws_role_name,
1007
+ error_str,
1008
+ )
1009
+ return self._auth_with_env_vars()
1010
+ # Genuine permission error — re-raise
1011
+ verbose_logger.error(
1012
+ "AssumeRole AccessDenied for %s and caller is NOT "
1013
+ "the same role. Re-raising. Error: %s",
1014
+ aws_role_name,
1015
+ error_str,
1016
+ )
1017
+ raise
1018
+
1019
+ # Extract the credentials from the response and convert to Session Credentials
1020
+ sts_credentials = sts_response["Credentials"]
1021
+ credentials = Credentials(
1022
+ access_key=sts_credentials["AccessKeyId"],
1023
+ secret_key=sts_credentials["SecretAccessKey"],
1024
+ token=sts_credentials["SessionToken"],
1025
+ )
1026
+
1027
+ sts_expiry = sts_credentials["Expiration"]
1028
+ # Convert to timezone-aware datetime for comparison
1029
+ current_time = datetime.now(sts_expiry.tzinfo)
1030
+ sts_ttl = (sts_expiry - current_time).total_seconds() - 60
1031
+ return credentials, sts_ttl
1032
+
1033
+ @tracer.wrap()
1034
+ def _auth_with_aws_profile(
1035
+ self, aws_profile_name: str
1036
+ ) -> Tuple[Credentials, Optional[int]]:
1037
+ """
1038
+ Authenticate with AWS profile
1039
+ """
1040
+ import boto3
1041
+
1042
+ # uses auth values from AWS profile usually stored in ~/.aws/credentials
1043
+ with tracer.trace("boto3.Session(profile_name=aws_profile_name)"):
1044
+ client = boto3.Session(profile_name=aws_profile_name)
1045
+ return client.get_credentials(), None
1046
+
1047
+ @tracer.wrap()
1048
+ def _auth_with_aws_session_token(
1049
+ self,
1050
+ aws_access_key_id: str,
1051
+ aws_secret_access_key: str,
1052
+ aws_session_token: str,
1053
+ ) -> Tuple[Credentials, Optional[int]]:
1054
+ """
1055
+ Authenticate with AWS Session Token
1056
+ """
1057
+ ### CHECK FOR AWS SESSION TOKEN ###
1058
+ from botocore.credentials import Credentials
1059
+
1060
+ credentials = Credentials(
1061
+ access_key=aws_access_key_id,
1062
+ secret_key=aws_secret_access_key,
1063
+ token=aws_session_token,
1064
+ )
1065
+
1066
+ return credentials, None
1067
+
1068
+ @tracer.wrap()
1069
+ def _auth_with_access_key_and_secret_key(
1070
+ self,
1071
+ aws_access_key_id: str,
1072
+ aws_secret_access_key: str,
1073
+ aws_region_name: Optional[str],
1074
+ ) -> Tuple[Credentials, Optional[int]]:
1075
+ """
1076
+ Authenticate with AWS Access Key and Secret Key
1077
+ """
1078
+ import boto3
1079
+
1080
+ # Check if credentials are already in cache. These credentials have no expiry time.
1081
+ with tracer.trace(
1082
+ "boto3.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name)"
1083
+ ):
1084
+ session = boto3.Session(
1085
+ aws_access_key_id=aws_access_key_id,
1086
+ aws_secret_access_key=aws_secret_access_key,
1087
+ region_name=aws_region_name,
1088
+ )
1089
+
1090
+ credentials = session.get_credentials()
1091
+ return credentials, self._get_default_ttl_for_boto3_credentials()
1092
+
1093
+ @tracer.wrap()
1094
+ def _auth_with_env_vars(self) -> Tuple[Credentials, Optional[int]]:
1095
+ """
1096
+ Authenticate with AWS Environment Variables
1097
+ """
1098
+ import boto3
1099
+
1100
+ with tracer.trace("boto3.Session()"):
1101
+ session = boto3.Session()
1102
+ credentials = session.get_credentials()
1103
+ return credentials, None
1104
+
1105
+ @tracer.wrap()
1106
+ def _get_default_ttl_for_boto3_credentials(self) -> int:
1107
+ """
1108
+ Get the default TTL for boto3 credentials
1109
+
1110
+ Returns `3600-60` which is 59 minutes
1111
+ """
1112
+ return 3600 - 60
1113
+
1114
+ def get_runtime_endpoint(
1115
+ self,
1116
+ api_base: Optional[str],
1117
+ aws_bedrock_runtime_endpoint: Optional[str],
1118
+ aws_region_name: str,
1119
+ endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]] = "runtime",
1120
+ ) -> Tuple[str, str]:
1121
+ env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT")
1122
+ if api_base is not None:
1123
+ endpoint_url = api_base
1124
+ elif aws_bedrock_runtime_endpoint is not None and isinstance(
1125
+ aws_bedrock_runtime_endpoint, str
1126
+ ):
1127
+ endpoint_url = aws_bedrock_runtime_endpoint
1128
+ elif env_aws_bedrock_runtime_endpoint and isinstance(
1129
+ env_aws_bedrock_runtime_endpoint, str
1130
+ ):
1131
+ endpoint_url = env_aws_bedrock_runtime_endpoint
1132
+ else:
1133
+ endpoint_url = self._select_default_endpoint_url(
1134
+ endpoint_type=endpoint_type,
1135
+ aws_region_name=aws_region_name,
1136
+ )
1137
+
1138
+ # Determine proxy_endpoint_url
1139
+ if aws_bedrock_runtime_endpoint is not None and isinstance(
1140
+ aws_bedrock_runtime_endpoint, str
1141
+ ):
1142
+ proxy_endpoint_url = aws_bedrock_runtime_endpoint
1143
+ elif env_aws_bedrock_runtime_endpoint and isinstance(
1144
+ env_aws_bedrock_runtime_endpoint, str
1145
+ ):
1146
+ proxy_endpoint_url = env_aws_bedrock_runtime_endpoint
1147
+ else:
1148
+ proxy_endpoint_url = endpoint_url
1149
+
1150
+ return endpoint_url, proxy_endpoint_url
1151
+
1152
+ def _select_default_endpoint_url(
1153
+ self,
1154
+ endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]],
1155
+ aws_region_name: str,
1156
+ ) -> str:
1157
+ """
1158
+ Select the default endpoint url based on the endpoint type
1159
+
1160
+ Default endpoint url is https://bedrock-runtime.{aws_region_name}.amazonaws.com
1161
+ """
1162
+ if endpoint_type == "agent":
1163
+ return f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com"
1164
+ elif endpoint_type == "agentcore":
1165
+ return f"https://bedrock-agentcore.{aws_region_name}.amazonaws.com"
1166
+ else:
1167
+ return f"https://bedrock-runtime.{aws_region_name}.amazonaws.com"
1168
+
1169
+ def _get_boto_credentials_from_optional_params(
1170
+ self, optional_params: dict, model: Optional[str] = None
1171
+ ) -> Boto3CredentialsInfo:
1172
+ """
1173
+ Get boto3 credentials from optional params
1174
+
1175
+ Args:
1176
+ optional_params (dict): Optional parameters for the model call
1177
+
1178
+ Returns:
1179
+ Credentials: Boto3 credentials object
1180
+ """
1181
+ try:
1182
+ from botocore.credentials import Credentials
1183
+ except ImportError:
1184
+ raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
1185
+ ## CREDENTIALS ##
1186
+ # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them
1187
+ aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
1188
+ aws_access_key_id = optional_params.pop("aws_access_key_id", None)
1189
+ aws_session_token = optional_params.pop("aws_session_token", None)
1190
+ aws_region_name = self._get_aws_region_name(optional_params, model)
1191
+ optional_params.pop("aws_region_name", None)
1192
+ aws_role_name = optional_params.pop("aws_role_name", None)
1193
+ aws_session_name = optional_params.pop("aws_session_name", None)
1194
+ aws_profile_name = optional_params.pop("aws_profile_name", None)
1195
+ aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
1196
+ aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
1197
+ aws_bedrock_runtime_endpoint = optional_params.pop(
1198
+ "aws_bedrock_runtime_endpoint", None
1199
+ ) # https://bedrock-runtime.{region_name}.amazonaws.com
1200
+ aws_external_id = optional_params.pop("aws_external_id", None)
1201
+
1202
+ credentials: Credentials = self.get_credentials(
1203
+ aws_access_key_id=aws_access_key_id,
1204
+ aws_secret_access_key=aws_secret_access_key,
1205
+ aws_session_token=aws_session_token,
1206
+ aws_region_name=aws_region_name,
1207
+ aws_session_name=aws_session_name,
1208
+ aws_profile_name=aws_profile_name,
1209
+ aws_role_name=aws_role_name,
1210
+ aws_web_identity_token=aws_web_identity_token,
1211
+ aws_sts_endpoint=aws_sts_endpoint,
1212
+ aws_external_id=aws_external_id,
1213
+ )
1214
+
1215
+ return Boto3CredentialsInfo(
1216
+ credentials=credentials,
1217
+ aws_region_name=aws_region_name,
1218
+ aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
1219
+ )
1220
+
1221
+ @tracer.wrap()
1222
+ def get_request_headers(
1223
+ self,
1224
+ credentials: Credentials,
1225
+ aws_region_name: str,
1226
+ extra_headers: Optional[dict],
1227
+ endpoint_url: str,
1228
+ data: Union[str, bytes],
1229
+ headers: dict,
1230
+ api_key: Optional[str] = None,
1231
+ ) -> AWSPreparedRequest:
1232
+ if api_key is not None:
1233
+ aws_bearer_token: Optional[str] = api_key
1234
+ else:
1235
+ aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
1236
+
1237
+ if aws_bearer_token:
1238
+ try:
1239
+ from botocore.awsrequest import AWSRequest
1240
+ except ImportError:
1241
+ raise ImportError(
1242
+ "Missing boto3 to call bedrock. Run 'pip install boto3'."
1243
+ )
1244
+ headers["Authorization"] = f"Bearer {aws_bearer_token}"
1245
+ request = AWSRequest(
1246
+ method="POST", url=endpoint_url, data=data, headers=headers
1247
+ )
1248
+ else:
1249
+ try:
1250
+ from botocore.auth import SigV4Auth
1251
+ from botocore.awsrequest import AWSRequest
1252
+ except ImportError:
1253
+ raise ImportError(
1254
+ "Missing boto3 to call bedrock. Run 'pip install boto3'."
1255
+ )
1256
+
1257
+ # Filter headers for AWS signature calculation
1258
+ # AWS SigV4 only includes specific headers in signature calculation
1259
+ aws_signature_headers = self._filter_headers_for_aws_signature(headers)
1260
+ sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name)
1261
+ request = AWSRequest(
1262
+ method="POST",
1263
+ url=endpoint_url,
1264
+ data=data,
1265
+ headers=aws_signature_headers,
1266
+ )
1267
+ sigv4.add_auth(request)
1268
+
1269
+ # Add back all original headers (including forwarded ones) after signature calculation
1270
+ for header_name, header_value in headers.items():
1271
+ if header_value is not None:
1272
+ request.headers[header_name] = header_value
1273
+
1274
+ if (
1275
+ extra_headers is not None and "Authorization" in extra_headers
1276
+ ): # prevent sigv4 from overwriting the auth header
1277
+ request.headers["Authorization"] = extra_headers["Authorization"]
1278
+ prepped = request.prepare()
1279
+
1280
+ return prepped
1281
+
1282
+ def _filter_headers_for_aws_signature(self, headers: dict) -> dict:
1283
+ """
1284
+ Filter headers to only include those that AWS SigV4 includes in signature calculation.
1285
+ This Fixes forwarded client headers from breaking the signature calculation.
1286
+ """
1287
+ aws_signature_headers = {}
1288
+ aws_headers = {
1289
+ "host",
1290
+ "content-type",
1291
+ "date",
1292
+ "x-amz-date",
1293
+ "x-amz-security-token",
1294
+ "x-amz-content-sha256",
1295
+ "x-amz-algorithm",
1296
+ "x-amz-credential",
1297
+ "x-amz-signedheaders",
1298
+ "x-amz-signature",
1299
+ }
1300
+
1301
+ for header_name, header_value in headers.items():
1302
+ if header_value is None:
1303
+ continue
1304
+ header_lower = header_name.lower()
1305
+ if (
1306
+ header_lower in aws_headers
1307
+ or header_lower.startswith("x-amz-")
1308
+ or header_lower.startswith("x-amzn-")
1309
+ ):
1310
+ aws_signature_headers[header_name] = header_value
1311
+
1312
+ return aws_signature_headers
1313
+
1314
+ def _sign_request(
1315
+ self,
1316
+ service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"],
1317
+ headers: dict,
1318
+ optional_params: dict,
1319
+ request_data: dict,
1320
+ api_base: str,
1321
+ model: Optional[str] = None,
1322
+ stream: Optional[bool] = None,
1323
+ fake_stream: Optional[bool] = None,
1324
+ api_key: Optional[str] = None,
1325
+ ) -> Tuple[dict, Optional[bytes]]:
1326
+ """
1327
+ Sign a request for Bedrock or Sagemaker
1328
+
1329
+ Returns:
1330
+ Tuple[dict, Optional[str]]: A tuple containing the headers and the json str body of the request
1331
+ """
1332
+ if api_key is not None:
1333
+ aws_bearer_token: Optional[str] = api_key
1334
+ else:
1335
+ aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
1336
+
1337
+ # If aws bearer token is set, use it directly in the header
1338
+ if aws_bearer_token:
1339
+ headers = headers or {}
1340
+ headers["Content-Type"] = "application/json"
1341
+ headers["Authorization"] = f"Bearer {aws_bearer_token}"
1342
+ return headers, json.dumps(request_data).encode()
1343
+
1344
+ # If no bearer token is set, proceed with the existing SigV4 authentication
1345
+ try:
1346
+ from botocore.auth import SigV4Auth
1347
+ from botocore.awsrequest import AWSRequest
1348
+ from botocore.credentials import Credentials
1349
+ except ImportError:
1350
+ raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
1351
+
1352
+ ## CREDENTIALS ##
1353
+ # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
1354
+ aws_secret_access_key = optional_params.get("aws_secret_access_key", None)
1355
+ aws_access_key_id = optional_params.get("aws_access_key_id", None)
1356
+ aws_session_token = optional_params.get("aws_session_token", None)
1357
+ aws_role_name = optional_params.get("aws_role_name", None)
1358
+ aws_session_name = optional_params.get("aws_session_name", None)
1359
+ aws_profile_name = optional_params.get("aws_profile_name", None)
1360
+ aws_web_identity_token = optional_params.get("aws_web_identity_token", None)
1361
+ aws_sts_endpoint = optional_params.get("aws_sts_endpoint", None)
1362
+ aws_external_id = optional_params.get("aws_external_id", None)
1363
+ aws_region_name = self._get_aws_region_name(
1364
+ optional_params=optional_params, model=model
1365
+ )
1366
+
1367
+ credentials: Credentials = self.get_credentials(
1368
+ aws_access_key_id=aws_access_key_id,
1369
+ aws_secret_access_key=aws_secret_access_key,
1370
+ aws_session_token=aws_session_token,
1371
+ aws_region_name=aws_region_name,
1372
+ aws_session_name=aws_session_name,
1373
+ aws_profile_name=aws_profile_name,
1374
+ aws_role_name=aws_role_name,
1375
+ aws_web_identity_token=aws_web_identity_token,
1376
+ aws_sts_endpoint=aws_sts_endpoint,
1377
+ aws_external_id=aws_external_id,
1378
+ )
1379
+
1380
+ sigv4 = SigV4Auth(credentials, service_name, aws_region_name)
1381
+ if headers is not None:
1382
+ headers = {"Content-Type": "application/json", **headers}
1383
+ else:
1384
+ headers = {"Content-Type": "application/json"}
1385
+
1386
+ aws_signature_headers = self._filter_headers_for_aws_signature(headers)
1387
+ request = AWSRequest(
1388
+ method="POST",
1389
+ url=api_base,
1390
+ data=json.dumps(request_data),
1391
+ headers=aws_signature_headers,
1392
+ )
1393
+ sigv4.add_auth(request)
1394
+
1395
+ request_headers_dict = dict(request.headers)
1396
+ # Add back original headers after signing. Only headers in SignedHeaders
1397
+ # are integrity-protected; forwarded headers (x-forwarded-*) must remain unsigned.
1398
+ for header_name, header_value in headers.items():
1399
+ if header_value is not None:
1400
+ request_headers_dict[header_name] = header_value
1401
+ if (
1402
+ headers is not None and "Authorization" in headers
1403
+ ): # prevent sigv4 from overwriting the auth header
1404
+ request_headers_dict["Authorization"] = headers["Authorization"]
1405
+
1406
+ return request_headers_dict, request.body
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/handler.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from openai.types.batch import BatchRequestCounts
2
+ from openai.types.batch import Metadata as OpenAIBatchMetadata
3
+
4
+ from litellm.types.utils import LiteLLMBatch
5
+
6
+
7
+ class BedrockBatchesHandler:
8
+ """
9
+ Handler for Bedrock Batches.
10
+
11
+ Specific providers/models needed some special handling.
12
+
13
+ E.g. Twelve Labs Embedding Async Invoke
14
+ """
15
+
16
+ @staticmethod
17
+ def _handle_async_invoke_status(
18
+ batch_id: str, aws_region_name: str, logging_obj=None, **kwargs
19
+ ) -> "LiteLLMBatch":
20
+ """
21
+ Handle async invoke status check for AWS Bedrock.
22
+
23
+ This is for Twelve Labs Embedding Async Invoke.
24
+
25
+ Args:
26
+ batch_id: The async invoke ARN
27
+ aws_region_name: AWS region name
28
+ **kwargs: Additional parameters
29
+
30
+ Returns:
31
+ dict: Status information including status, output_file_id (S3 URL), etc.
32
+ """
33
+ import asyncio
34
+
35
+ from litellm.llms.bedrock.embed.embedding import BedrockEmbedding
36
+
37
+ async def _async_get_status():
38
+ # Create embedding handler instance
39
+ embedding_handler = BedrockEmbedding()
40
+
41
+ # Get the status of the async invoke job
42
+ status_response = await embedding_handler._get_async_invoke_status(
43
+ invocation_arn=batch_id,
44
+ aws_region_name=aws_region_name,
45
+ logging_obj=logging_obj,
46
+ **kwargs,
47
+ )
48
+
49
+ # Transform response to a LiteLLMBatch object
50
+ from litellm.types.utils import LiteLLMBatch
51
+
52
+ openai_batch_metadata: OpenAIBatchMetadata = {
53
+ "output_file_id": status_response["outputDataConfig"][
54
+ "s3OutputDataConfig"
55
+ ]["s3Uri"],
56
+ "failure_message": status_response.get("failureMessage") or "",
57
+ "model_arn": status_response["modelArn"],
58
+ }
59
+
60
+ result = LiteLLMBatch(
61
+ id=status_response["invocationArn"],
62
+ object="batch",
63
+ status=status_response["status"],
64
+ created_at=status_response["submitTime"],
65
+ in_progress_at=status_response["lastModifiedTime"],
66
+ completed_at=status_response.get("endTime"),
67
+ failed_at=status_response.get("endTime")
68
+ if status_response["status"] == "failed"
69
+ else None,
70
+ request_counts=BatchRequestCounts(
71
+ total=1,
72
+ completed=1 if status_response["status"] == "completed" else 0,
73
+ failed=1 if status_response["status"] == "failed" else 0,
74
+ ),
75
+ metadata=openai_batch_metadata,
76
+ completion_window="24h",
77
+ endpoint="/v1/embeddings",
78
+ input_file_id="",
79
+ )
80
+
81
+ return result
82
+
83
+ # Since this function is called from within an async context via run_in_executor,
84
+ # we need to create a new event loop in a thread to avoid conflicts
85
+ import concurrent.futures
86
+
87
+ def run_in_thread():
88
+ new_loop = asyncio.new_event_loop()
89
+ asyncio.set_event_loop(new_loop)
90
+ try:
91
+ return new_loop.run_until_complete(_async_get_status())
92
+ finally:
93
+ new_loop.close()
94
+
95
+ with concurrent.futures.ThreadPoolExecutor() as executor:
96
+ future = executor.submit(run_in_thread)
97
+ return future.result()
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/transformation.py ADDED
@@ -0,0 +1,549 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import time
3
+ from typing import Any, Dict, List, Literal, Optional, Union, cast
4
+
5
+ from httpx import Headers, Response
6
+
7
+ from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
8
+ from litellm.llms.base_llm.chat.transformation import BaseLLMException
9
+ from litellm.secret_managers.main import get_secret_str
10
+ from litellm.types.llms.bedrock import (
11
+ BedrockCreateBatchRequest,
12
+ BedrockCreateBatchResponse,
13
+ BedrockInputDataConfig,
14
+ BedrockOutputDataConfig,
15
+ BedrockS3InputDataConfig,
16
+ BedrockS3OutputDataConfig,
17
+ )
18
+ from litellm.types.llms.openai import (
19
+ AllMessageValues,
20
+ CreateBatchRequest,
21
+ )
22
+ from litellm.types.utils import LiteLLMBatch, LlmProviders
23
+
24
+ from ..base_aws_llm import BaseAWSLLM
25
+ from ..common_utils import CommonBatchFilesUtils
26
+
27
+
28
+ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
29
+ """
30
+ Config for Bedrock Batches - handles batch job creation and management for Bedrock
31
+ """
32
+
33
+ def __init__(self):
34
+ super().__init__()
35
+ self.common_utils = CommonBatchFilesUtils()
36
+
37
+ @property
38
+ def custom_llm_provider(self) -> LlmProviders:
39
+ return LlmProviders.BEDROCK
40
+
41
+ def validate_environment(
42
+ self,
43
+ headers: dict,
44
+ model: str,
45
+ messages: List[AllMessageValues],
46
+ optional_params: dict,
47
+ litellm_params: dict,
48
+ api_key: Optional[str] = None,
49
+ api_base: Optional[str] = None,
50
+ ) -> dict:
51
+ """
52
+ Validate and prepare environment for Bedrock batch requests.
53
+ AWS credentials are handled by BaseAWSLLM.
54
+ """
55
+ # Add any Bedrock-specific headers if needed
56
+ return headers
57
+
58
+ def get_complete_batch_url(
59
+ self,
60
+ api_base: Optional[str],
61
+ api_key: Optional[str],
62
+ model: str,
63
+ optional_params: Dict,
64
+ litellm_params: Dict,
65
+ data: CreateBatchRequest,
66
+ ) -> str:
67
+ """
68
+ Get the complete URL for Bedrock batch creation.
69
+ Bedrock batch jobs are created via the model invocation job API.
70
+ """
71
+ aws_region_name = self._get_aws_region_name(optional_params, model)
72
+
73
+ # Bedrock model invocation job endpoint
74
+ # Format: https://bedrock.{region}.amazonaws.com/model-invocation-job
75
+ bedrock_endpoint = (
76
+ f"https://bedrock.{aws_region_name}.amazonaws.com/model-invocation-job"
77
+ )
78
+
79
+ return bedrock_endpoint
80
+
81
+ def transform_create_batch_request(
82
+ self,
83
+ model: str,
84
+ create_batch_data: CreateBatchRequest,
85
+ optional_params: dict,
86
+ litellm_params: dict,
87
+ ) -> Dict[str, Any]:
88
+ """
89
+ Transform the batch creation request to Bedrock format.
90
+
91
+ Bedrock batch inference requires:
92
+ - modelId: The Bedrock model ID
93
+ - jobName: Unique name for the batch job
94
+ - inputDataConfig: Configuration for input data (S3 location)
95
+ - outputDataConfig: Configuration for output data (S3 location)
96
+ - roleArn: IAM role ARN for the batch job
97
+ """
98
+ # Get required parameters
99
+ input_file_id = create_batch_data.get("input_file_id")
100
+ if not input_file_id:
101
+ raise ValueError("input_file_id is required for Bedrock batch creation")
102
+
103
+ # Extract S3 information from file ID using common utility
104
+ input_bucket, input_key = self.common_utils.parse_s3_uri(input_file_id)
105
+
106
+ # Get output S3 configuration
107
+ output_bucket = litellm_params.get("s3_output_bucket_name") or os.getenv(
108
+ "AWS_S3_OUTPUT_BUCKET_NAME"
109
+ )
110
+ if not output_bucket:
111
+ # Use same bucket as input if no output bucket specified
112
+ output_bucket = input_bucket
113
+
114
+ # Get IAM role ARN
115
+ role_arn = (
116
+ litellm_params.get("aws_batch_role_arn")
117
+ or optional_params.get("aws_batch_role_arn")
118
+ or os.getenv("AWS_BATCH_ROLE_ARN")
119
+ )
120
+ if not role_arn:
121
+ raise ValueError(
122
+ "AWS IAM role ARN is required for Bedrock batch jobs. "
123
+ "Set 'aws_batch_role_arn' in litellm_params or AWS_BATCH_ROLE_ARN env var"
124
+ )
125
+
126
+ if not model:
127
+ raise ValueError(
128
+ "Could not determine Bedrock model ID. Please pass `model` in your request body."
129
+ )
130
+
131
+ # Generate job name with the correct model ID using common utility
132
+ job_name = self.common_utils.generate_unique_job_name(model, prefix="litellm")
133
+ output_key = f"litellm-batch-outputs/{job_name}/"
134
+
135
+ # Build input data config
136
+ input_data_config: BedrockInputDataConfig = {
137
+ "s3InputDataConfig": BedrockS3InputDataConfig(
138
+ s3Uri=f"s3://{input_bucket}/{input_key}"
139
+ )
140
+ }
141
+
142
+ # Build output data config
143
+ s3_output_config: BedrockS3OutputDataConfig = BedrockS3OutputDataConfig(
144
+ s3Uri=f"s3://{output_bucket}/{output_key}"
145
+ )
146
+
147
+ # Add optional KMS encryption key ID if provided
148
+ s3_encryption_key_id = litellm_params.get(
149
+ "s3_encryption_key_id"
150
+ ) or get_secret_str("AWS_S3_ENCRYPTION_KEY_ID")
151
+ if s3_encryption_key_id:
152
+ s3_output_config["s3EncryptionKeyId"] = s3_encryption_key_id
153
+
154
+ output_data_config: BedrockOutputDataConfig = {
155
+ "s3OutputDataConfig": s3_output_config
156
+ }
157
+
158
+ # Create Bedrock batch request with proper typing
159
+ bedrock_request: BedrockCreateBatchRequest = {
160
+ "modelId": model,
161
+ "jobName": job_name,
162
+ "inputDataConfig": input_data_config,
163
+ "outputDataConfig": output_data_config,
164
+ "roleArn": role_arn,
165
+ }
166
+
167
+ # Add optional parameters if provided
168
+ completion_window = create_batch_data.get("completion_window")
169
+ if completion_window:
170
+ # Map OpenAI completion window to Bedrock timeout
171
+ # OpenAI uses "24h", Bedrock expects timeout in hours
172
+ if completion_window == "24h":
173
+ bedrock_request["timeoutDurationInHours"] = 24
174
+
175
+ # For Bedrock, we need to return a pre-signed request with AWS auth headers
176
+ # Use common utility for AWS signing
177
+ endpoint_url = f"https://bedrock.{self._get_aws_region_name(optional_params, model)}.amazonaws.com/model-invocation-job"
178
+ signed_headers, signed_data = self.common_utils.sign_aws_request(
179
+ service_name="bedrock",
180
+ data=bedrock_request,
181
+ endpoint_url=endpoint_url,
182
+ optional_params=optional_params,
183
+ method="POST",
184
+ )
185
+
186
+ # Return a pre-signed request format that the HTTP handler can use
187
+ return {
188
+ "method": "POST",
189
+ "url": endpoint_url,
190
+ "headers": signed_headers,
191
+ "data": signed_data.decode("utf-8"),
192
+ }
193
+
194
+ def transform_create_batch_response(
195
+ self,
196
+ model: Optional[str],
197
+ raw_response: Response,
198
+ logging_obj: Any,
199
+ litellm_params: dict,
200
+ ) -> LiteLLMBatch:
201
+ """
202
+ Transform Bedrock batch creation response to LiteLLM format.
203
+ """
204
+ try:
205
+ response_data: BedrockCreateBatchResponse = raw_response.json()
206
+ except Exception as e:
207
+ raise ValueError(f"Failed to parse Bedrock batch response: {e}")
208
+
209
+ # Extract information from typed Bedrock response
210
+ job_arn = response_data.get("jobArn", "")
211
+ status_str: str = str(response_data.get("status", "Submitted"))
212
+
213
+ # Map Bedrock status to OpenAI-compatible status
214
+ status_mapping: Dict[str, str] = {
215
+ "Submitted": "validating",
216
+ "Validating": "validating",
217
+ "Scheduled": "in_progress",
218
+ "InProgress": "in_progress",
219
+ "PartiallyCompleted": "completed",
220
+ "Completed": "completed",
221
+ "Failed": "failed",
222
+ "Stopping": "cancelling",
223
+ "Stopped": "cancelled",
224
+ "Expired": "expired",
225
+ }
226
+
227
+ openai_status = cast(
228
+ Literal[
229
+ "validating",
230
+ "failed",
231
+ "in_progress",
232
+ "finalizing",
233
+ "completed",
234
+ "expired",
235
+ "cancelling",
236
+ "cancelled",
237
+ ],
238
+ status_mapping.get(status_str, "validating"),
239
+ )
240
+
241
+ # Get original request data from litellm_params if available
242
+ original_request = litellm_params.get("original_batch_request", {})
243
+
244
+ # Create LiteLLM batch object
245
+ return LiteLLMBatch(
246
+ id=job_arn, # Use ARN as the batch ID
247
+ object="batch",
248
+ endpoint=original_request.get("endpoint", "/v1/chat/completions"),
249
+ errors=None,
250
+ input_file_id=original_request.get("input_file_id", ""),
251
+ completion_window=original_request.get("completion_window", "24h"),
252
+ status=openai_status,
253
+ output_file_id=None, # Will be populated when job completes
254
+ error_file_id=None,
255
+ created_at=int(time.time()),
256
+ in_progress_at=int(time.time()) if status_str == "InProgress" else None,
257
+ expires_at=None,
258
+ finalizing_at=None,
259
+ completed_at=None,
260
+ failed_at=None,
261
+ expired_at=None,
262
+ cancelling_at=None,
263
+ cancelled_at=None,
264
+ request_counts=None,
265
+ metadata=original_request.get("metadata", {}),
266
+ )
267
+
268
+ def transform_retrieve_batch_request(
269
+ self,
270
+ batch_id: str,
271
+ optional_params: dict,
272
+ litellm_params: dict,
273
+ ) -> Dict[str, Any]:
274
+ """
275
+ Transform batch retrieval request for Bedrock.
276
+
277
+ Args:
278
+ batch_id: Bedrock job ARN
279
+ optional_params: Optional parameters
280
+ litellm_params: LiteLLM parameters
281
+
282
+ Returns:
283
+ Transformed request data for Bedrock GetModelInvocationJob API
284
+ """
285
+ # For Bedrock, batch_id should be the full job ARN
286
+ # The GetModelInvocationJob API expects the full ARN as the identifier
287
+ if not batch_id.startswith("arn:aws:bedrock:"):
288
+ raise ValueError(f"Invalid batch_id format. Expected ARN, got: {batch_id}")
289
+
290
+ # Extract the job identifier from the ARN - use the full ARN path part
291
+ # ARN format: arn:aws:bedrock:region:account:model-invocation-job/job-name
292
+ arn_parts = batch_id.split(":")
293
+ if len(arn_parts) < 6:
294
+ raise ValueError(f"Invalid ARN format: {batch_id}")
295
+
296
+ region = arn_parts[3]
297
+ # arn_parts[5] contains "model-invocation-job/{jobId}"
298
+
299
+ # Build the endpoint URL for GetModelInvocationJob
300
+ # AWS API format: GET /model-invocation-job/{jobIdentifier}
301
+ # Use the FULL ARN as jobIdentifier and URL-encode it (includes ':' and '/')
302
+ import urllib.parse as _ul
303
+
304
+ encoded_arn = _ul.quote(batch_id, safe="")
305
+ endpoint_url = (
306
+ f"https://bedrock.{region}.amazonaws.com/model-invocation-job/{encoded_arn}"
307
+ )
308
+
309
+ # Use common utility for AWS signing
310
+ signed_headers, _ = self.common_utils.sign_aws_request(
311
+ service_name="bedrock",
312
+ data={}, # GET request has no body
313
+ endpoint_url=endpoint_url,
314
+ optional_params=optional_params,
315
+ method="GET",
316
+ )
317
+
318
+ # Return pre-signed request format
319
+ return {
320
+ "method": "GET",
321
+ "url": endpoint_url,
322
+ "headers": signed_headers,
323
+ "data": None,
324
+ }
325
+
326
+ def _parse_timestamps_and_status(self, response_data, status_str: str):
327
+ """Helper to parse timestamps based on status."""
328
+ import datetime
329
+
330
+ def parse_timestamp(ts_str: Optional[str]) -> Optional[int]:
331
+ if not ts_str:
332
+ return None
333
+ try:
334
+ dt = datetime.datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
335
+ return int(dt.timestamp())
336
+ except Exception:
337
+ return None
338
+
339
+ created_at = parse_timestamp(
340
+ str(response_data.get("submitTime"))
341
+ if response_data.get("submitTime") is not None
342
+ else None
343
+ )
344
+ in_progress_states = {"InProgress", "Validating", "Scheduled"}
345
+ in_progress_at = (
346
+ parse_timestamp(
347
+ str(response_data.get("lastModifiedTime"))
348
+ if response_data.get("lastModifiedTime") is not None
349
+ else None
350
+ )
351
+ if status_str in in_progress_states
352
+ else None
353
+ )
354
+ completed_at = (
355
+ parse_timestamp(
356
+ str(response_data.get("endTime"))
357
+ if response_data.get("endTime") is not None
358
+ else None
359
+ )
360
+ if status_str in {"Completed", "PartiallyCompleted"}
361
+ else None
362
+ )
363
+ failed_at = (
364
+ parse_timestamp(
365
+ str(response_data.get("endTime"))
366
+ if response_data.get("endTime") is not None
367
+ else None
368
+ )
369
+ if status_str == "Failed"
370
+ else None
371
+ )
372
+ cancelled_at = (
373
+ parse_timestamp(
374
+ str(response_data.get("endTime"))
375
+ if response_data.get("endTime") is not None
376
+ else None
377
+ )
378
+ if status_str == "Stopped"
379
+ else None
380
+ )
381
+ expires_at = parse_timestamp(
382
+ str(response_data.get("jobExpirationTime"))
383
+ if response_data.get("jobExpirationTime") is not None
384
+ else None
385
+ )
386
+
387
+ return (
388
+ created_at,
389
+ in_progress_at,
390
+ completed_at,
391
+ failed_at,
392
+ cancelled_at,
393
+ expires_at,
394
+ )
395
+
396
+ def _extract_file_configs(self, response_data):
397
+ """Helper to extract input and output file configurations."""
398
+ # Extract input file ID
399
+ input_file_id = ""
400
+ input_data_config = response_data.get("inputDataConfig", {})
401
+ if isinstance(input_data_config, dict):
402
+ s3_input_config = input_data_config.get("s3InputDataConfig", {})
403
+ if isinstance(s3_input_config, dict):
404
+ input_file_id = s3_input_config.get("s3Uri", "")
405
+
406
+ # Extract output file ID
407
+ output_file_id = None
408
+ output_data_config = response_data.get("outputDataConfig", {})
409
+ if isinstance(output_data_config, dict):
410
+ s3_output_config = output_data_config.get("s3OutputDataConfig", {})
411
+ if isinstance(s3_output_config, dict):
412
+ output_file_id = s3_output_config.get("s3Uri", "")
413
+
414
+ return input_file_id, output_file_id
415
+
416
+ def _extract_errors_and_metadata(self, response_data, raw_response):
417
+ """Helper to extract errors and enriched metadata."""
418
+ # Extract errors
419
+ message = response_data.get("message")
420
+ errors = None
421
+ if message:
422
+ from openai.types.batch import Errors
423
+ from openai.types.batch_error import BatchError
424
+
425
+ errors = Errors(
426
+ data=[BatchError(message=message, code=str(raw_response.status_code))],
427
+ object="list",
428
+ )
429
+
430
+ # Enrich metadata with useful Bedrock fields
431
+ enriched_metadata_raw: Dict[str, Any] = {
432
+ "jobName": response_data.get("jobName"),
433
+ "clientRequestToken": response_data.get("clientRequestToken"),
434
+ "modelId": response_data.get("modelId"),
435
+ "roleArn": response_data.get("roleArn"),
436
+ "timeoutDurationInHours": response_data.get("timeoutDurationInHours"),
437
+ "vpcConfig": response_data.get("vpcConfig"),
438
+ }
439
+ import json as _json
440
+
441
+ enriched_metadata: Dict[str, str] = {}
442
+ for _k, _v in enriched_metadata_raw.items():
443
+ if _v is None:
444
+ continue
445
+ if isinstance(_v, (dict, list)):
446
+ try:
447
+ enriched_metadata[_k] = _json.dumps(_v)
448
+ except Exception:
449
+ enriched_metadata[_k] = str(_v)
450
+ else:
451
+ enriched_metadata[_k] = str(_v)
452
+
453
+ return errors, enriched_metadata
454
+
455
+ def transform_retrieve_batch_response(
456
+ self,
457
+ model: Optional[str],
458
+ raw_response: Response,
459
+ logging_obj: Any,
460
+ litellm_params: dict,
461
+ ) -> LiteLLMBatch:
462
+ """
463
+ Transform Bedrock batch retrieval response to LiteLLM format.
464
+ """
465
+ from litellm.types.llms.bedrock import BedrockGetBatchResponse
466
+
467
+ try:
468
+ response_data: BedrockGetBatchResponse = raw_response.json()
469
+ except Exception as e:
470
+ raise ValueError(f"Failed to parse Bedrock batch response: {e}")
471
+
472
+ job_arn = response_data.get("jobArn", "")
473
+ status_str: str = str(response_data.get("status", "Submitted"))
474
+
475
+ # Map Bedrock status to OpenAI-compatible status
476
+ status_mapping: Dict[str, str] = {
477
+ "Submitted": "validating",
478
+ "Validating": "validating",
479
+ "Scheduled": "in_progress",
480
+ "InProgress": "in_progress",
481
+ "PartiallyCompleted": "completed",
482
+ "Completed": "completed",
483
+ "Failed": "failed",
484
+ "Stopping": "cancelling",
485
+ "Stopped": "cancelled",
486
+ "Expired": "expired",
487
+ }
488
+ openai_status = cast(
489
+ Literal[
490
+ "validating",
491
+ "failed",
492
+ "in_progress",
493
+ "finalizing",
494
+ "completed",
495
+ "expired",
496
+ "cancelling",
497
+ "cancelled",
498
+ ],
499
+ status_mapping.get(status_str, "validating"),
500
+ )
501
+
502
+ # Parse timestamps
503
+ (
504
+ created_at,
505
+ in_progress_at,
506
+ completed_at,
507
+ failed_at,
508
+ cancelled_at,
509
+ expires_at,
510
+ ) = self._parse_timestamps_and_status(response_data, status_str)
511
+
512
+ # Extract file configurations
513
+ input_file_id, output_file_id = self._extract_file_configs(response_data)
514
+
515
+ # Extract errors and metadata
516
+ errors, enriched_metadata = self._extract_errors_and_metadata(
517
+ response_data, raw_response
518
+ )
519
+
520
+ return LiteLLMBatch(
521
+ id=job_arn,
522
+ object="batch",
523
+ endpoint="/v1/chat/completions",
524
+ errors=errors,
525
+ input_file_id=input_file_id,
526
+ completion_window="24h",
527
+ status=openai_status,
528
+ output_file_id=output_file_id,
529
+ error_file_id=None,
530
+ created_at=created_at or int(time.time()),
531
+ in_progress_at=in_progress_at,
532
+ expires_at=expires_at,
533
+ finalizing_at=None,
534
+ completed_at=completed_at,
535
+ failed_at=failed_at,
536
+ expired_at=None,
537
+ cancelling_at=None,
538
+ cancelled_at=cancelled_at,
539
+ request_counts=None,
540
+ metadata=enriched_metadata,
541
+ )
542
+
543
+ def get_error_class(
544
+ self, error_message: str, status_code: int, headers: Union[Dict, Headers]
545
+ ) -> BaseLLMException:
546
+ """
547
+ Get Bedrock-specific error class using common utility.
548
+ """
549
+ return self.common_utils.get_error_class(error_message, status_code, headers)
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/__init__.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+
3
+ from .converse_handler import BedrockConverseLLM
4
+ from .invoke_handler import (
5
+ AmazonAnthropicClaudeStreamDecoder,
6
+ AmazonDeepSeekR1StreamDecoder,
7
+ AWSEventStreamDecoder,
8
+ BedrockLLM,
9
+ )
10
+
11
+
12
+ def get_bedrock_event_stream_decoder(
13
+ invoke_provider: Optional[str], model: str, sync_stream: bool, json_mode: bool
14
+ ):
15
+ if invoke_provider and invoke_provider == "anthropic":
16
+ decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder(
17
+ model=model,
18
+ sync_stream=sync_stream,
19
+ json_mode=json_mode,
20
+ )
21
+ return decoder
22
+ elif invoke_provider and invoke_provider == "deepseek_r1":
23
+ decoder = AmazonDeepSeekR1StreamDecoder(
24
+ model=model,
25
+ sync_stream=sync_stream,
26
+ )
27
+ return decoder
28
+ else:
29
+ decoder = AWSEventStreamDecoder(model=model)
30
+ return decoder
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ from .transformation import AmazonAgentCoreConfig
2
+
3
+ __all__ = ["AmazonAgentCoreConfig"]
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/transformation.py ADDED
@@ -0,0 +1,1037 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation for Bedrock AgentCore
3
+
4
+ https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html
5
+ """
6
+
7
+ import json
8
+ from collections.abc import AsyncGenerator
9
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
10
+ from urllib.parse import quote
11
+
12
+ import httpx
13
+
14
+ from litellm._logging import verbose_logger
15
+ from litellm._uuid import uuid
16
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
17
+ convert_content_list_to_str,
18
+ )
19
+ from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
20
+ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
21
+ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
22
+ from litellm.llms.bedrock.common_utils import BedrockError
23
+ from litellm.types.llms.bedrock_agentcore import (
24
+ AgentCoreMessage,
25
+ AgentCoreParsedResponse,
26
+ AgentCoreUsage,
27
+ )
28
+ from litellm.types.llms.openai import AllMessageValues
29
+ from litellm.types.utils import (
30
+ Choices,
31
+ Delta,
32
+ Message,
33
+ ModelResponse,
34
+ ModelResponseStream,
35
+ StreamingChoices,
36
+ Usage,
37
+ )
38
+
39
+ if TYPE_CHECKING:
40
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
41
+ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
42
+
43
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
44
+ else:
45
+ LiteLLMLoggingObj = Any
46
+ HTTPHandler = Any
47
+ AsyncHTTPHandler = Any
48
+
49
+
50
+ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
51
+ def __init__(self, **kwargs):
52
+ BaseConfig.__init__(self, **kwargs)
53
+ BaseAWSLLM.__init__(self, **kwargs)
54
+
55
+ def get_supported_openai_params(self, model: str) -> List[str]:
56
+ """
57
+ Bedrock AgentCore has 0 OpenAI compatible params
58
+ """
59
+ return []
60
+
61
+ def map_openai_params(
62
+ self,
63
+ non_default_params: dict,
64
+ optional_params: dict,
65
+ model: str,
66
+ drop_params: bool,
67
+ ) -> dict:
68
+ """
69
+ Map OpenAI params to AgentCore params
70
+ """
71
+ return optional_params
72
+
73
+ def get_complete_url(
74
+ self,
75
+ api_base: Optional[str],
76
+ api_key: Optional[str],
77
+ model: str,
78
+ optional_params: dict,
79
+ litellm_params: dict,
80
+ stream: Optional[bool] = None,
81
+ ) -> str:
82
+ """
83
+ Get the complete url for the request
84
+ """
85
+ ### SET RUNTIME ENDPOINT ###
86
+ aws_bedrock_runtime_endpoint = optional_params.get(
87
+ "aws_bedrock_runtime_endpoint", None
88
+ )
89
+
90
+ # Extract ARN from model string
91
+ agent_runtime_arn = self._get_agent_runtime_arn(model)
92
+
93
+ # Parse ARN to get region
94
+ region = self._extract_region_from_arn(agent_runtime_arn)
95
+
96
+ # Build the base endpoint URL for AgentCore
97
+ # Note: We don't use get_runtime_endpoint as AgentCore has its own endpoint structure
98
+ if aws_bedrock_runtime_endpoint:
99
+ base_url = aws_bedrock_runtime_endpoint
100
+ else:
101
+ base_url = f"https://bedrock-agentcore.{region}.amazonaws.com"
102
+
103
+ # Based on boto3 client.invoke_agent_runtime, the path is:
104
+ # /runtimes/{URL-ENCODED-ARN}/invocations?qualifier=<value>
105
+ encoded_arn = quote(agent_runtime_arn, safe="")
106
+ endpoint_url = f"{base_url}/runtimes/{encoded_arn}/invocations"
107
+
108
+ # Add qualifier as query parameter if provided
109
+ if "qualifier" in optional_params:
110
+ endpoint_url = f"{endpoint_url}?qualifier={optional_params['qualifier']}"
111
+
112
+ return endpoint_url
113
+
114
+ def sign_request(
115
+ self,
116
+ headers: dict,
117
+ optional_params: dict,
118
+ request_data: dict,
119
+ api_base: str,
120
+ api_key: Optional[str] = None,
121
+ model: Optional[str] = None,
122
+ stream: Optional[bool] = None,
123
+ fake_stream: Optional[bool] = None,
124
+ ) -> Tuple[dict, Optional[bytes]]:
125
+ # Set Accept header required by MCP servers on AgentCore
126
+ # Per MCP spec (Streamable HTTP transport): client MUST include Accept header
127
+ # listing both application/json and text/event-stream as supported content types
128
+ headers["Accept"] = "application/json, text/event-stream"
129
+
130
+ # Check if api_key (bearer token) is provided for Cognito authentication
131
+ # Priority: api_key parameter first, then optional_params
132
+ jwt_token = api_key or optional_params.get("api_key")
133
+ if jwt_token:
134
+ verbose_logger.debug(
135
+ f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..."
136
+ )
137
+ headers["Content-Type"] = "application/json"
138
+ headers["Authorization"] = f"Bearer {jwt_token}"
139
+ # Return headers with bearer token and JSON-encoded body (not SigV4 signed)
140
+ return headers, json.dumps(request_data).encode()
141
+
142
+ # Otherwise, use AWS SigV4 authentication
143
+ verbose_logger.debug("AgentCore: Using AWS SigV4 authentication (IAM)")
144
+ return self._sign_request(
145
+ service_name="bedrock-agentcore",
146
+ headers=headers,
147
+ optional_params=optional_params,
148
+ request_data=request_data,
149
+ api_base=api_base,
150
+ model=model,
151
+ stream=stream,
152
+ fake_stream=fake_stream,
153
+ api_key=api_key,
154
+ )
155
+
156
+ def _get_agent_runtime_arn(self, model: str) -> str:
157
+ """
158
+ Extract ARN from model string
159
+ model = "agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC"
160
+ returns: "arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC"
161
+ """
162
+ parts = model.split("/", 1)
163
+ if len(parts) != 2 or parts[0] != "agentcore":
164
+ raise ValueError(
165
+ "Invalid model format. Expected format: 'model=bedrock/agentcore/arn:aws:bedrock-agentcore:region:account:runtime/runtime_id'"
166
+ )
167
+ return parts[1]
168
+
169
+ def _extract_region_from_arn(self, arn: str) -> str:
170
+ """
171
+ Extract region from ARN
172
+ arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC
173
+ returns: us-west-2
174
+ """
175
+ parts = arn.split(":")
176
+ if len(parts) >= 4:
177
+ return parts[3]
178
+ raise ValueError(f"Invalid ARN format: {arn}")
179
+
180
+ def _get_runtime_session_id(self, optional_params: dict) -> str:
181
+ """
182
+ Get or generate runtime session ID (must be 33+ chars)
183
+ """
184
+ session_id = optional_params.get("runtimeSessionId", None)
185
+ if session_id:
186
+ verbose_logger.debug(f"Using provided runtimeSessionId: {session_id}")
187
+ return session_id
188
+
189
+ # Generate a session ID with 33+ characters
190
+ generated_id = f"litellm-session-{str(uuid.uuid4())}"
191
+ verbose_logger.debug(f"Generated new session ID: {generated_id}")
192
+ return generated_id
193
+
194
+ def _get_runtime_user_id(self, optional_params: dict) -> Optional[str]:
195
+ """
196
+ Get runtime user ID if provided
197
+ """
198
+ user_id = optional_params.get("runtimeUserId", None)
199
+ if user_id:
200
+ verbose_logger.debug(f"Using provided runtimeUserId: {user_id}")
201
+ return user_id
202
+
203
+ def transform_request(
204
+ self,
205
+ model: str,
206
+ messages: List[AllMessageValues],
207
+ optional_params: dict,
208
+ litellm_params: dict,
209
+ headers: dict,
210
+ ) -> dict:
211
+ """
212
+ Transform the request to AgentCore format.
213
+
214
+ Based on boto3's implementation:
215
+ - Session ID goes in header: X-Amzn-Bedrock-AgentCore-Runtime-Session-Id
216
+ - User ID goes in header: X-Amzn-Bedrock-AgentCore-Runtime-User-Id
217
+ - Qualifier goes as query parameter
218
+ - Only the payload goes in the request body
219
+
220
+ Returns:
221
+ dict: Payload dict containing the prompt
222
+ """
223
+ verbose_logger.debug(
224
+ f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}"
225
+ )
226
+
227
+ # Use the last message content as the prompt
228
+ prompt = convert_content_list_to_str(messages[-1])
229
+
230
+ # Create the payload - this is what goes in the body (raw JSON)
231
+ payload: dict = {"prompt": prompt}
232
+
233
+ # Get or generate session ID - this goes in the header
234
+ runtime_session_id = self._get_runtime_session_id(optional_params)
235
+ headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = runtime_session_id
236
+
237
+ # Get user ID if provided - this goes in the header
238
+ runtime_user_id = self._get_runtime_user_id(optional_params)
239
+ if runtime_user_id:
240
+ headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id
241
+
242
+ # The request data is the payload dict (will be JSON encoded by the HTTP handler)
243
+ # Qualifier will be handled as a query parameter in get_complete_url
244
+
245
+ verbose_logger.debug(f"PAYLOAD: {payload}")
246
+ return payload
247
+
248
+ def _extract_sse_json(self, line: str) -> Optional[Dict]:
249
+ """Extract and parse JSON from an SSE data line."""
250
+ if not line.startswith("data:"):
251
+ return None
252
+
253
+ json_str = line[5:].strip()
254
+ if not json_str:
255
+ return None
256
+
257
+ try:
258
+ data = json.loads(json_str)
259
+ # Skip non-dict data (some lines contain JSON strings)
260
+ return data if isinstance(data, dict) else None
261
+ except json.JSONDecodeError:
262
+ verbose_logger.debug(f"Skipping non-JSON line: {line[:100]}")
263
+ return None
264
+
265
+ def _extract_usage_from_event(self, event_data: Dict) -> Optional[AgentCoreUsage]:
266
+ """Extract usage information from event metadata."""
267
+ event_payload = event_data.get("event")
268
+ if not event_payload:
269
+ return None
270
+
271
+ metadata = event_payload.get("metadata")
272
+ if metadata and "usage" in metadata:
273
+ return metadata["usage"] # type: ignore
274
+
275
+ return None
276
+
277
+ def _extract_content_delta(self, event_data: Dict) -> Optional[str]:
278
+ """Extract text content from contentBlockDelta event."""
279
+ event_payload = event_data.get("event")
280
+ if not event_payload:
281
+ return None
282
+
283
+ content_block_delta = event_payload.get("contentBlockDelta")
284
+ if not content_block_delta:
285
+ return None
286
+
287
+ delta = content_block_delta.get("delta", {})
288
+ return delta.get("text")
289
+
290
+ def _extract_content_from_message(self, message: AgentCoreMessage) -> str:
291
+ """
292
+ Extract text content from message content blocks.
293
+ This works for both SSE messages and JSON responses.
294
+ """
295
+ content_list = message.get("content", [])
296
+ if not isinstance(content_list, list):
297
+ return ""
298
+
299
+ return "".join(
300
+ block["text"]
301
+ for block in content_list
302
+ if isinstance(block, dict) and "text" in block
303
+ )
304
+
305
+ def _calculate_usage(
306
+ self, model: str, messages: List[AllMessageValues], content: str
307
+ ) -> Optional[Usage]:
308
+ """
309
+ Calculate token usage using LiteLLM's token counter.
310
+
311
+ Args:
312
+ model: The model name
313
+ messages: Input messages
314
+ content: Response content
315
+
316
+ Returns:
317
+ Usage object with calculated tokens, or None if calculation fails
318
+ """
319
+ try:
320
+ from litellm.utils import token_counter
321
+
322
+ prompt_tokens = token_counter(model=model, messages=messages)
323
+ completion_tokens = token_counter(
324
+ model=model, text=content, count_response_tokens=True
325
+ )
326
+ total_tokens = prompt_tokens + completion_tokens
327
+
328
+ verbose_logger.debug(
329
+ f"Calculated usage - prompt: {prompt_tokens}, completion: {completion_tokens}, total: {total_tokens}"
330
+ )
331
+
332
+ return Usage(
333
+ prompt_tokens=prompt_tokens,
334
+ completion_tokens=completion_tokens,
335
+ total_tokens=total_tokens,
336
+ )
337
+ except Exception as e:
338
+ verbose_logger.warning(f"Failed to calculate token usage: {str(e)}")
339
+ return None
340
+
341
+ def _parse_json_response(self, response_json: dict) -> AgentCoreParsedResponse:
342
+ """
343
+ Parse direct JSON response (non-streaming).
344
+
345
+ Supports multiple agent response schemas:
346
+ 1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore
347
+ 2. {"response": [{"text": "..."}]} - Strands agent format
348
+ 3. {"result": "plain text"} or {"response": "plain text"} - simple string
349
+ 4. Fallback: raw JSON as content string
350
+ """
351
+ # Guard: if json.loads() returned a non-dict (e.g. array or primitive),
352
+ # skip strategy matching and fall back to raw JSON string
353
+ if not isinstance(response_json, dict):
354
+ verbose_logger.warning(
355
+ "AgentCore: JSON response is not a dict. "
356
+ "Returning raw JSON as content."
357
+ )
358
+ return AgentCoreParsedResponse(
359
+ content=json.dumps(response_json),
360
+ usage=None,
361
+ final_message=None,
362
+ )
363
+
364
+ # Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format
365
+ if "result" in response_json and isinstance(response_json["result"], dict):
366
+ result = response_json["result"]
367
+ content = self._extract_content_from_message(result) # type: ignore
368
+ return AgentCoreParsedResponse(
369
+ content=content,
370
+ usage=None,
371
+ final_message=result, # type: ignore
372
+ )
373
+
374
+ # Strategy 2: {"response": [{"text": "..."}]} - Strands agent content blocks
375
+ if "response" in response_json and isinstance(response_json["response"], list):
376
+ content = self._extract_content_from_message(
377
+ {"content": response_json["response"]} # type: ignore
378
+ )
379
+ return AgentCoreParsedResponse(
380
+ content=content,
381
+ usage=None,
382
+ final_message=None,
383
+ )
384
+
385
+ # Strategy 3: string values - {"result": "text"} or {"response": "text"}
386
+ for key in ("result", "response"):
387
+ val = response_json.get(key)
388
+ if isinstance(val, str):
389
+ return AgentCoreParsedResponse(
390
+ content=val,
391
+ usage=None,
392
+ final_message=None,
393
+ )
394
+
395
+ # Strategy 4: fallback - return raw JSON as content
396
+ verbose_logger.warning(
397
+ f"AgentCore: Could not extract content from JSON response keys "
398
+ f"{list(response_json.keys())}. Returning raw JSON as content."
399
+ )
400
+ return AgentCoreParsedResponse(
401
+ content=json.dumps(response_json),
402
+ usage=None,
403
+ final_message=None,
404
+ )
405
+
406
+ def _get_parsed_response(
407
+ self, raw_response: httpx.Response
408
+ ) -> AgentCoreParsedResponse:
409
+ """
410
+ Parse AgentCore response based on content type.
411
+
412
+ Args:
413
+ raw_response: Raw HTTP response from AgentCore
414
+
415
+ Returns:
416
+ AgentCoreParsedResponse: Parsed response data
417
+ """
418
+ content_type = raw_response.headers.get("content-type", "").lower()
419
+ verbose_logger.debug(f"AgentCore response Content-Type: {content_type}")
420
+
421
+ # Parse response based on content type
422
+ if "application/json" in content_type:
423
+ # Direct JSON response
424
+ verbose_logger.debug("Parsing JSON response")
425
+ response_json = raw_response.json()
426
+ verbose_logger.debug(f"Response JSON: {response_json}")
427
+ return self._parse_json_response(response_json)
428
+ else:
429
+ # SSE stream response (text/event-stream or default)
430
+ verbose_logger.debug("Parsing SSE stream response")
431
+ response_text = raw_response.text
432
+ verbose_logger.debug(
433
+ f"AgentCore response (first 500 chars): {response_text[:500]}"
434
+ )
435
+ return self._parse_sse_stream(response_text)
436
+
437
+ def _parse_sse_stream(self, response_text: str) -> AgentCoreParsedResponse:
438
+ """
439
+ Parse Server-Sent Events (SSE) stream format.
440
+ Each line starts with 'data:' followed by JSON.
441
+
442
+ Returns:
443
+ AgentCoreParsedResponse: Parsed response with content, usage, and message
444
+ """
445
+ final_message: Optional[AgentCoreMessage] = None
446
+ usage_data: Optional[AgentCoreUsage] = None
447
+ content_blocks: List[str] = []
448
+
449
+ for line in response_text.strip().split("\n"):
450
+ line = line.strip()
451
+ if not line:
452
+ continue
453
+
454
+ data = self._extract_sse_json(line)
455
+ if not data:
456
+ continue
457
+
458
+ verbose_logger.debug(f"SSE event keys: {list(data.keys())}")
459
+
460
+ # Check for final complete message
461
+ if "message" in data and isinstance(data["message"], dict):
462
+ final_message = data["message"] # type: ignore
463
+ verbose_logger.debug("Found final message")
464
+
465
+ # Process event data
466
+ if "event" in data and isinstance(data["event"], dict):
467
+ event_payload = data["event"]
468
+ verbose_logger.debug(
469
+ f"Event payload keys: {list(event_payload.keys())}"
470
+ )
471
+
472
+ # Extract usage metadata
473
+ if usage := self._extract_usage_from_event(data):
474
+ usage_data = usage
475
+ verbose_logger.debug(f"Found usage data: {usage_data}")
476
+
477
+ # Collect content deltas
478
+ if text := self._extract_content_delta(data):
479
+ content_blocks.append(text)
480
+
481
+ # Build final content
482
+ content = (
483
+ self._extract_content_from_message(final_message)
484
+ if final_message
485
+ else "".join(content_blocks)
486
+ )
487
+
488
+ verbose_logger.debug(f"Final usage_data: {usage_data}")
489
+
490
+ return AgentCoreParsedResponse(
491
+ content=content, usage=usage_data, final_message=final_message
492
+ )
493
+
494
+ def _stream_agentcore_response_sync(
495
+ self,
496
+ response: httpx.Response,
497
+ model: str,
498
+ ):
499
+ """
500
+ Internal sync generator that parses SSE and yields ModelResponse chunks.
501
+ """
502
+ buffer = ""
503
+ for text_chunk in response.iter_text():
504
+ buffer += text_chunk
505
+
506
+ # Process complete lines
507
+ while "\n" in buffer:
508
+ line, buffer = buffer.split("\n", 1)
509
+ line = line.strip()
510
+
511
+ if not line or not line.startswith("data:"):
512
+ continue
513
+
514
+ json_str = line[5:].strip()
515
+ if not json_str:
516
+ continue
517
+
518
+ try:
519
+ data_obj = json.loads(json_str)
520
+ if not isinstance(data_obj, dict):
521
+ continue
522
+
523
+ # Process contentBlockDelta events
524
+ if "event" in data_obj and isinstance(data_obj["event"], dict):
525
+ event_payload = data_obj["event"]
526
+ content_block_delta = event_payload.get("contentBlockDelta")
527
+
528
+ if content_block_delta:
529
+ delta = content_block_delta.get("delta", {})
530
+ text = delta.get("text", "")
531
+
532
+ if text:
533
+ chunk = ModelResponseStream(
534
+ id=f"chatcmpl-{uuid.uuid4()}",
535
+ created=0,
536
+ model=model,
537
+ object="chat.completion.chunk",
538
+ )
539
+ chunk.choices = [
540
+ StreamingChoices(
541
+ finish_reason=None,
542
+ index=0,
543
+ delta=Delta(content=text, role="assistant"),
544
+ )
545
+ ]
546
+ yield chunk
547
+
548
+ # Process metadata/usage
549
+ metadata = event_payload.get("metadata")
550
+ if metadata and "usage" in metadata:
551
+ chunk = ModelResponseStream(
552
+ id=f"chatcmpl-{uuid.uuid4()}",
553
+ created=0,
554
+ model=model,
555
+ object="chat.completion.chunk",
556
+ )
557
+ chunk.choices = [
558
+ StreamingChoices(
559
+ finish_reason="stop",
560
+ index=0,
561
+ delta=Delta(),
562
+ )
563
+ ]
564
+ usage_data: AgentCoreUsage = metadata["usage"] # type: ignore
565
+ setattr(
566
+ chunk,
567
+ "usage",
568
+ Usage(
569
+ prompt_tokens=usage_data.get("inputTokens", 0),
570
+ completion_tokens=usage_data.get("outputTokens", 0),
571
+ total_tokens=usage_data.get("totalTokens", 0),
572
+ ),
573
+ )
574
+ yield chunk
575
+
576
+ # Process final message
577
+ if "message" in data_obj and isinstance(data_obj["message"], dict):
578
+ chunk = ModelResponseStream(
579
+ id=f"chatcmpl-{uuid.uuid4()}",
580
+ created=0,
581
+ model=model,
582
+ object="chat.completion.chunk",
583
+ )
584
+ chunk.choices = [
585
+ StreamingChoices(
586
+ finish_reason="stop",
587
+ index=0,
588
+ delta=Delta(),
589
+ )
590
+ ]
591
+ yield chunk
592
+
593
+ except json.JSONDecodeError:
594
+ verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
595
+ continue
596
+
597
+ def get_sync_custom_stream_wrapper(
598
+ self,
599
+ model: str,
600
+ custom_llm_provider: str,
601
+ logging_obj: LiteLLMLoggingObj,
602
+ api_base: str,
603
+ headers: dict,
604
+ data: dict,
605
+ messages: list,
606
+ client: Optional[Union[HTTPHandler, "AsyncHTTPHandler"]] = None,
607
+ json_mode: Optional[bool] = None,
608
+ signed_json_body: Optional[bytes] = None,
609
+ ) -> "CustomStreamWrapper":
610
+ """
611
+ Simplified sync streaming - returns a generator that yields ModelResponse chunks.
612
+ """
613
+ from litellm.llms.custom_httpx.http_handler import (
614
+ HTTPHandler,
615
+ _get_httpx_client,
616
+ )
617
+
618
+ if client is None or not isinstance(client, HTTPHandler):
619
+ client = _get_httpx_client(params={})
620
+
621
+ verbose_logger.debug(f"Making sync streaming request to: {api_base}")
622
+
623
+ # Make streaming request
624
+ response = client.post(
625
+ api_base,
626
+ headers=headers,
627
+ data=signed_json_body if signed_json_body else json.dumps(data),
628
+ stream=True,
629
+ logging_obj=logging_obj,
630
+ )
631
+
632
+ if response.status_code != 200:
633
+ raise BedrockError(
634
+ status_code=response.status_code, message=str(response.read())
635
+ )
636
+
637
+ # LOGGING
638
+ logging_obj.post_call(
639
+ input=messages,
640
+ api_key="",
641
+ original_response="first stream response received",
642
+ additional_args={"complete_input_dict": data},
643
+ )
644
+
645
+ # Check if response is JSON (agent used sync return) instead of SSE
646
+ content_type = response.headers.get("content-type", "").lower()
647
+ if "application/json" in content_type:
648
+ verbose_logger.debug(
649
+ "AgentCore streaming: received JSON response instead of SSE, "
650
+ "converting to single-chunk stream"
651
+ )
652
+ try:
653
+ body = response.read()
654
+ response_json = json.loads(body)
655
+ except (json.JSONDecodeError, Exception) as e:
656
+ raise BedrockError(
657
+ status_code=response.status_code,
658
+ message=f"AgentCore: Failed to read/parse JSON response body: {e}",
659
+ )
660
+ parsed = self._parse_json_response(response_json)
661
+
662
+ def _json_as_sync_stream():
663
+ # Content chunk
664
+ content_chunk = ModelResponseStream(
665
+ id=f"chatcmpl-{uuid.uuid4()}",
666
+ created=0,
667
+ model=model,
668
+ object="chat.completion.chunk",
669
+ )
670
+ content_chunk.choices = [
671
+ StreamingChoices(
672
+ finish_reason=None,
673
+ index=0,
674
+ delta=Delta(content=parsed["content"], role="assistant"),
675
+ )
676
+ ]
677
+ yield content_chunk
678
+
679
+ # Stop sentinel chunk (matches SSE path convention)
680
+ stop_chunk = ModelResponseStream(
681
+ id=f"chatcmpl-{uuid.uuid4()}",
682
+ created=0,
683
+ model=model,
684
+ object="chat.completion.chunk",
685
+ )
686
+ stop_chunk.choices = [
687
+ StreamingChoices(
688
+ finish_reason="stop",
689
+ index=0,
690
+ delta=Delta(),
691
+ )
692
+ ]
693
+ yield stop_chunk
694
+
695
+ return CustomStreamWrapper(
696
+ completion_stream=_json_as_sync_stream(),
697
+ model=model,
698
+ custom_llm_provider="bedrock",
699
+ logging_obj=logging_obj,
700
+ )
701
+
702
+ # SSE stream (text/event-stream or default) - use existing SSE parser
703
+ return CustomStreamWrapper(
704
+ completion_stream=self._stream_agentcore_response_sync(response, model),
705
+ model=model,
706
+ custom_llm_provider="bedrock",
707
+ logging_obj=logging_obj,
708
+ )
709
+
710
+ async def _stream_agentcore_response(
711
+ self,
712
+ response: httpx.Response,
713
+ model: str,
714
+ ) -> AsyncGenerator[ModelResponseStream, None]:
715
+ """
716
+ Internal async generator that parses SSE and yields ModelResponse chunks.
717
+ """
718
+ buffer = ""
719
+ async for text_chunk in response.aiter_text():
720
+ buffer += text_chunk
721
+
722
+ # Process complete lines
723
+ while "\n" in buffer:
724
+ line, buffer = buffer.split("\n", 1)
725
+ line = line.strip()
726
+
727
+ if not line or not line.startswith("data:"):
728
+ continue
729
+
730
+ json_str = line[5:].strip()
731
+ if not json_str:
732
+ continue
733
+
734
+ try:
735
+ data_obj = json.loads(json_str)
736
+ if not isinstance(data_obj, dict):
737
+ continue
738
+
739
+ # Process contentBlockDelta events
740
+ if "event" in data_obj and isinstance(data_obj["event"], dict):
741
+ event_payload = data_obj["event"]
742
+ content_block_delta = event_payload.get("contentBlockDelta")
743
+
744
+ if content_block_delta:
745
+ delta = content_block_delta.get("delta", {})
746
+ text = delta.get("text", "")
747
+
748
+ if text:
749
+ chunk = ModelResponseStream(
750
+ id=f"chatcmpl-{uuid.uuid4()}",
751
+ created=0,
752
+ model=model,
753
+ object="chat.completion.chunk",
754
+ )
755
+ chunk.choices = [
756
+ StreamingChoices(
757
+ finish_reason=None,
758
+ index=0,
759
+ delta=Delta(content=text, role="assistant"),
760
+ )
761
+ ]
762
+ yield chunk
763
+
764
+ # Process metadata/usage
765
+ metadata = event_payload.get("metadata")
766
+ if metadata and "usage" in metadata:
767
+ chunk = ModelResponseStream(
768
+ id=f"chatcmpl-{uuid.uuid4()}",
769
+ created=0,
770
+ model=model,
771
+ object="chat.completion.chunk",
772
+ )
773
+ chunk.choices = [
774
+ StreamingChoices(
775
+ finish_reason="stop",
776
+ index=0,
777
+ delta=Delta(),
778
+ )
779
+ ]
780
+ usage_data: AgentCoreUsage = metadata["usage"] # type: ignore
781
+ setattr(
782
+ chunk,
783
+ "usage",
784
+ Usage(
785
+ prompt_tokens=usage_data.get("inputTokens", 0),
786
+ completion_tokens=usage_data.get("outputTokens", 0),
787
+ total_tokens=usage_data.get("totalTokens", 0),
788
+ ),
789
+ )
790
+ yield chunk
791
+
792
+ # Process final message
793
+ if "message" in data_obj and isinstance(data_obj["message"], dict):
794
+ chunk = ModelResponseStream(
795
+ id=f"chatcmpl-{uuid.uuid4()}",
796
+ created=0,
797
+ model=model,
798
+ object="chat.completion.chunk",
799
+ )
800
+ chunk.choices = [
801
+ StreamingChoices(
802
+ finish_reason="stop",
803
+ index=0,
804
+ delta=Delta(),
805
+ )
806
+ ]
807
+ yield chunk
808
+
809
+ except json.JSONDecodeError:
810
+ verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
811
+ continue
812
+
813
+ async def get_async_custom_stream_wrapper(
814
+ self,
815
+ model: str,
816
+ custom_llm_provider: str,
817
+ logging_obj: LiteLLMLoggingObj,
818
+ api_base: str,
819
+ headers: dict,
820
+ data: dict,
821
+ messages: list,
822
+ client: Optional["AsyncHTTPHandler"] = None,
823
+ json_mode: Optional[bool] = None,
824
+ signed_json_body: Optional[bytes] = None,
825
+ ) -> "CustomStreamWrapper":
826
+ """
827
+ Simplified async streaming - returns an async generator that yields ModelResponse chunks.
828
+ """
829
+ from litellm.llms.custom_httpx.http_handler import (
830
+ AsyncHTTPHandler,
831
+ get_async_httpx_client,
832
+ )
833
+
834
+ if client is None or not isinstance(client, AsyncHTTPHandler):
835
+ client = get_async_httpx_client(
836
+ llm_provider=cast(Any, "bedrock"), params={}
837
+ )
838
+
839
+ verbose_logger.debug(f"Making async streaming request to: {api_base}")
840
+
841
+ # Make async streaming request
842
+ response = await client.post(
843
+ api_base,
844
+ headers=headers,
845
+ data=signed_json_body if signed_json_body else json.dumps(data),
846
+ stream=True,
847
+ logging_obj=logging_obj,
848
+ )
849
+
850
+ if response.status_code != 200:
851
+ raise BedrockError(
852
+ status_code=response.status_code, message=str(await response.aread())
853
+ )
854
+
855
+ # LOGGING
856
+ logging_obj.post_call(
857
+ input=messages,
858
+ api_key="",
859
+ original_response="first stream response received",
860
+ additional_args={"complete_input_dict": data},
861
+ )
862
+
863
+ # Check if response is JSON (agent used sync return) instead of SSE
864
+ content_type = response.headers.get("content-type", "").lower()
865
+ if "application/json" in content_type:
866
+ verbose_logger.debug(
867
+ "AgentCore streaming: received JSON response instead of SSE, "
868
+ "converting to single-chunk stream"
869
+ )
870
+ try:
871
+ body = await response.aread()
872
+ response_json = json.loads(body)
873
+ except (json.JSONDecodeError, Exception) as e:
874
+ raise BedrockError(
875
+ status_code=response.status_code,
876
+ message=f"AgentCore: Failed to read/parse JSON response body: {e}",
877
+ )
878
+ parsed = self._parse_json_response(response_json)
879
+
880
+ async def _json_as_async_stream() -> AsyncGenerator[
881
+ ModelResponseStream, None
882
+ ]:
883
+ # Content chunk
884
+ content_chunk = ModelResponseStream(
885
+ id=f"chatcmpl-{uuid.uuid4()}",
886
+ created=0,
887
+ model=model,
888
+ object="chat.completion.chunk",
889
+ )
890
+ content_chunk.choices = [
891
+ StreamingChoices(
892
+ finish_reason=None,
893
+ index=0,
894
+ delta=Delta(content=parsed["content"], role="assistant"),
895
+ )
896
+ ]
897
+ yield content_chunk
898
+
899
+ # Stop sentinel chunk (matches SSE path convention)
900
+ stop_chunk = ModelResponseStream(
901
+ id=f"chatcmpl-{uuid.uuid4()}",
902
+ created=0,
903
+ model=model,
904
+ object="chat.completion.chunk",
905
+ )
906
+ stop_chunk.choices = [
907
+ StreamingChoices(
908
+ finish_reason="stop",
909
+ index=0,
910
+ delta=Delta(),
911
+ )
912
+ ]
913
+ yield stop_chunk
914
+
915
+ return CustomStreamWrapper(
916
+ completion_stream=_json_as_async_stream(),
917
+ model=model,
918
+ custom_llm_provider="bedrock",
919
+ logging_obj=logging_obj,
920
+ )
921
+
922
+ # SSE stream (text/event-stream or default) - use existing SSE parser
923
+ return CustomStreamWrapper(
924
+ completion_stream=self._stream_agentcore_response(response, model),
925
+ model=model,
926
+ custom_llm_provider="bedrock",
927
+ logging_obj=logging_obj,
928
+ )
929
+
930
+ @property
931
+ def has_custom_stream_wrapper(self) -> bool:
932
+ """Indicates that this config has custom streaming support."""
933
+ return True
934
+
935
+ @property
936
+ def supports_stream_param_in_request_body(self) -> bool:
937
+ """
938
+ AgentCore does not allow passing `stream` in the request body.
939
+ Streaming is automatic based on the response format.
940
+ """
941
+ return False
942
+
943
+ def transform_response(
944
+ self,
945
+ model: str,
946
+ raw_response: httpx.Response,
947
+ model_response: ModelResponse,
948
+ logging_obj: LiteLLMLoggingObj,
949
+ request_data: dict,
950
+ messages: List[AllMessageValues],
951
+ optional_params: dict,
952
+ litellm_params: dict,
953
+ encoding: Any,
954
+ api_key: Optional[str] = None,
955
+ json_mode: Optional[bool] = None,
956
+ ) -> ModelResponse:
957
+ """
958
+ Transform the AgentCore response to LiteLLM ModelResponse format.
959
+ AgentCore can return either JSON or SSE (Server-Sent Events) stream responses.
960
+
961
+ Note: For streaming responses, use get_streaming_response() instead.
962
+ """
963
+ try:
964
+ # Parse the response based on content type (JSON or SSE)
965
+ parsed_data = self._get_parsed_response(raw_response)
966
+
967
+ content = parsed_data["content"]
968
+ usage_data = parsed_data["usage"]
969
+
970
+ verbose_logger.debug(f"Parsed content length: {len(content)}")
971
+ verbose_logger.debug(f"Usage data: {usage_data}")
972
+
973
+ # Create the message
974
+ message = Message(content=content, role="assistant")
975
+
976
+ # Create choices
977
+ choice = Choices(finish_reason="stop", index=0, message=message)
978
+
979
+ # Update model response
980
+ model_response.choices = [choice]
981
+ model_response.model = model
982
+
983
+ # Add usage information if available
984
+ # Note: AgentCore JSON responses don't include usage data
985
+ # SSE responses may include usage in metadata events
986
+ if usage_data:
987
+ usage = Usage(
988
+ prompt_tokens=usage_data.get("inputTokens", 0),
989
+ completion_tokens=usage_data.get("outputTokens", 0),
990
+ total_tokens=usage_data.get("totalTokens", 0),
991
+ )
992
+ setattr(model_response, "usage", usage)
993
+ else:
994
+ # Calculate token usage using LiteLLM's token counter
995
+ verbose_logger.debug(
996
+ "No usage data from AgentCore - calculating tokens"
997
+ )
998
+ calculated_usage = self._calculate_usage(model, messages, content)
999
+ if calculated_usage:
1000
+ setattr(model_response, "usage", calculated_usage)
1001
+
1002
+ return model_response
1003
+
1004
+ except Exception as e:
1005
+ verbose_logger.error(
1006
+ f"Error processing Bedrock AgentCore response: {str(e)}"
1007
+ )
1008
+ raise BedrockError(
1009
+ message=f"Error processing response: {str(e)}",
1010
+ status_code=raw_response.status_code,
1011
+ )
1012
+
1013
+ def validate_environment(
1014
+ self,
1015
+ headers: dict,
1016
+ model: str,
1017
+ messages: List[AllMessageValues],
1018
+ optional_params: dict,
1019
+ litellm_params: dict,
1020
+ api_key: Optional[str] = None,
1021
+ api_base: Optional[str] = None,
1022
+ ) -> dict:
1023
+ return headers
1024
+
1025
+ def get_error_class(
1026
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
1027
+ ) -> BaseLLMException:
1028
+ return BedrockError(status_code=status_code, message=error_message)
1029
+
1030
+ def should_fake_stream(
1031
+ self,
1032
+ model: Optional[str],
1033
+ stream: Optional[bool],
1034
+ custom_llm_provider: Optional[str] = None,
1035
+ ) -> bool:
1036
+ # AgentCore supports true streaming - don't buffer
1037
+ return False
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_handler.py ADDED
@@ -0,0 +1,512 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from typing import Any, Optional, Union
3
+
4
+ import httpx
5
+
6
+ import litellm
7
+ from litellm.anthropic_beta_headers_manager import (
8
+ update_headers_with_filtered_beta,
9
+ )
10
+ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
11
+ from litellm.llms.custom_httpx.http_handler import (
12
+ AsyncHTTPHandler,
13
+ HTTPHandler,
14
+ _get_httpx_client,
15
+ get_async_httpx_client,
16
+ )
17
+ from litellm.types.utils import ModelResponse
18
+ from litellm.utils import CustomStreamWrapper
19
+
20
+ from ..base_aws_llm import BaseAWSLLM, Credentials
21
+ from ..common_utils import BedrockError, _get_all_bedrock_regions
22
+ from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
23
+
24
+
25
+ def make_sync_call(
26
+ client: Optional[HTTPHandler],
27
+ api_base: str,
28
+ headers: dict,
29
+ data: str,
30
+ model: str,
31
+ messages: list,
32
+ logging_obj: LiteLLMLoggingObject,
33
+ json_mode: Optional[bool] = False,
34
+ fake_stream: bool = False,
35
+ stream_chunk_size: int = 1024,
36
+ ):
37
+ if client is None:
38
+ client = _get_httpx_client() # Create a new client if none provided
39
+
40
+ response = client.post(
41
+ api_base,
42
+ headers=headers,
43
+ data=data,
44
+ stream=not fake_stream,
45
+ logging_obj=logging_obj,
46
+ )
47
+
48
+ if response.status_code != 200:
49
+ raise BedrockError(
50
+ status_code=response.status_code, message=str(response.read())
51
+ )
52
+
53
+ if fake_stream:
54
+ model_response: (
55
+ ModelResponse
56
+ ) = litellm.AmazonConverseConfig()._transform_response(
57
+ model=model,
58
+ response=response,
59
+ model_response=litellm.ModelResponse(),
60
+ stream=True,
61
+ logging_obj=logging_obj,
62
+ optional_params={},
63
+ api_key="",
64
+ data=data,
65
+ messages=messages,
66
+ encoding=litellm.encoding,
67
+ ) # type: ignore
68
+ completion_stream: Any = MockResponseIterator(
69
+ model_response=model_response, json_mode=json_mode
70
+ )
71
+ else:
72
+ decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
73
+ completion_stream = decoder.iter_bytes(
74
+ response.iter_bytes(chunk_size=stream_chunk_size)
75
+ )
76
+
77
+ # LOGGING
78
+ logging_obj.post_call(
79
+ input=messages,
80
+ api_key="",
81
+ original_response="first stream response received",
82
+ additional_args={"complete_input_dict": data},
83
+ )
84
+
85
+ return completion_stream
86
+
87
+
88
+ class BedrockConverseLLM(BaseAWSLLM):
89
+ def __init__(self) -> None:
90
+ super().__init__()
91
+
92
+ async def async_streaming(
93
+ self,
94
+ model: str,
95
+ messages: list,
96
+ api_base: str,
97
+ model_response: ModelResponse,
98
+ timeout: Optional[Union[float, httpx.Timeout]],
99
+ encoding,
100
+ logging_obj,
101
+ stream,
102
+ optional_params: dict,
103
+ litellm_params: dict,
104
+ credentials: Credentials,
105
+ logger_fn=None,
106
+ headers={},
107
+ client: Optional[AsyncHTTPHandler] = None,
108
+ fake_stream: bool = False,
109
+ json_mode: Optional[bool] = False,
110
+ api_key: Optional[str] = None,
111
+ stream_chunk_size: int = 1024,
112
+ ) -> CustomStreamWrapper:
113
+ request_data = await litellm.AmazonConverseConfig()._async_transform_request(
114
+ model=model,
115
+ messages=messages,
116
+ optional_params=optional_params,
117
+ litellm_params=litellm_params,
118
+ headers=headers,
119
+ )
120
+ data = json.dumps(request_data)
121
+
122
+ prepped = self.get_request_headers(
123
+ credentials=credentials,
124
+ aws_region_name=litellm_params.get("aws_region_name") or "us-west-2",
125
+ extra_headers=headers,
126
+ endpoint_url=api_base,
127
+ data=data,
128
+ headers=headers,
129
+ api_key=api_key,
130
+ )
131
+
132
+ ## LOGGING
133
+ logging_obj.pre_call(
134
+ input=messages,
135
+ api_key="",
136
+ additional_args={
137
+ "complete_input_dict": data,
138
+ "api_base": api_base,
139
+ "headers": dict(prepped.headers),
140
+ },
141
+ )
142
+
143
+ completion_stream = await make_call(
144
+ client=client,
145
+ api_base=api_base,
146
+ headers=dict(prepped.headers),
147
+ data=data,
148
+ model=model,
149
+ messages=messages,
150
+ logging_obj=logging_obj,
151
+ fake_stream=fake_stream,
152
+ json_mode=json_mode,
153
+ stream_chunk_size=stream_chunk_size,
154
+ )
155
+ streaming_response = CustomStreamWrapper(
156
+ completion_stream=completion_stream,
157
+ model=model,
158
+ custom_llm_provider="bedrock",
159
+ logging_obj=logging_obj,
160
+ )
161
+ return streaming_response
162
+
163
+ async def async_completion(
164
+ self,
165
+ model: str,
166
+ messages: list,
167
+ api_base: str,
168
+ model_response: ModelResponse,
169
+ timeout: Optional[Union[float, httpx.Timeout]],
170
+ encoding,
171
+ logging_obj: LiteLLMLoggingObject,
172
+ stream,
173
+ optional_params: dict,
174
+ litellm_params: dict,
175
+ credentials: Credentials,
176
+ logger_fn=None,
177
+ headers: dict = {},
178
+ client: Optional[AsyncHTTPHandler] = None,
179
+ api_key: Optional[str] = None,
180
+ ) -> Union[ModelResponse, CustomStreamWrapper]:
181
+ request_data = await litellm.AmazonConverseConfig()._async_transform_request(
182
+ model=model,
183
+ messages=messages,
184
+ optional_params=optional_params,
185
+ litellm_params=litellm_params,
186
+ headers=headers,
187
+ )
188
+ data = json.dumps(request_data)
189
+
190
+ prepped = self.get_request_headers(
191
+ credentials=credentials,
192
+ aws_region_name=litellm_params.get("aws_region_name") or "us-west-2",
193
+ extra_headers=headers,
194
+ endpoint_url=api_base,
195
+ data=data,
196
+ headers=headers,
197
+ api_key=api_key,
198
+ )
199
+
200
+ ## LOGGING
201
+ logging_obj.pre_call(
202
+ input=messages,
203
+ api_key="",
204
+ additional_args={
205
+ "complete_input_dict": data,
206
+ "api_base": api_base,
207
+ "headers": prepped.headers,
208
+ },
209
+ )
210
+
211
+ headers = dict(prepped.headers)
212
+ if client is None or not isinstance(client, AsyncHTTPHandler):
213
+ _params = {}
214
+ if timeout is not None:
215
+ if isinstance(timeout, float) or isinstance(timeout, int):
216
+ timeout = httpx.Timeout(timeout)
217
+ _params["timeout"] = timeout
218
+ client = get_async_httpx_client(
219
+ params=_params, llm_provider=litellm.LlmProviders.BEDROCK
220
+ )
221
+ else:
222
+ client = client # type: ignore
223
+
224
+ try:
225
+ response = await client.post(
226
+ url=api_base,
227
+ headers=headers,
228
+ data=data,
229
+ logging_obj=logging_obj,
230
+ ) # type: ignore
231
+ response.raise_for_status()
232
+ except httpx.HTTPStatusError as err:
233
+ error_code = err.response.status_code
234
+ raise BedrockError(status_code=error_code, message=err.response.text)
235
+ except httpx.TimeoutException:
236
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
237
+
238
+ return litellm.AmazonConverseConfig()._transform_response(
239
+ model=model,
240
+ response=response,
241
+ model_response=model_response,
242
+ stream=stream if isinstance(stream, bool) else False,
243
+ logging_obj=logging_obj,
244
+ api_key="",
245
+ data=data,
246
+ messages=messages,
247
+ optional_params=optional_params,
248
+ encoding=encoding,
249
+ )
250
+
251
+ def completion( # noqa: PLR0915
252
+ self,
253
+ model: str,
254
+ messages: list,
255
+ api_base: Optional[str],
256
+ custom_prompt_dict: dict,
257
+ model_response: ModelResponse,
258
+ encoding,
259
+ logging_obj: LiteLLMLoggingObject,
260
+ optional_params: dict,
261
+ acompletion: bool,
262
+ timeout: Optional[Union[float, httpx.Timeout]],
263
+ litellm_params: dict,
264
+ logger_fn=None,
265
+ extra_headers: Optional[dict] = None,
266
+ client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None,
267
+ api_key: Optional[str] = None,
268
+ ):
269
+ ## SETUP ##
270
+ stream = optional_params.pop("stream", None)
271
+ stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
272
+ unencoded_model_id = optional_params.pop("model_id", None)
273
+ fake_stream = optional_params.pop("fake_stream", False)
274
+ json_mode = optional_params.get("json_mode", False)
275
+ if unencoded_model_id is not None:
276
+ modelId = self.encode_model_id(model_id=unencoded_model_id)
277
+ else:
278
+ # Strip nova spec prefixes before encoding model ID for API URL
279
+ _model_for_id = model
280
+ _stripped = _model_for_id
281
+ for rp in ["bedrock/converse/", "bedrock/", "converse/"]:
282
+ if _stripped.startswith(rp):
283
+ _stripped = _stripped[len(rp) :]
284
+ break
285
+ # Strip embedded region prefix (e.g. "bedrock/us-east-1/model" -> "model")
286
+ # and capture it so it can be used as aws_region_name below.
287
+ _region_from_model: Optional[str] = None
288
+ _potential_region = _stripped.split("/", 1)[0]
289
+ if _potential_region in _get_all_bedrock_regions() and "/" in _stripped:
290
+ _region_from_model = _potential_region
291
+ _stripped = _stripped.split("/", 1)[1]
292
+ _model_for_id = _stripped
293
+ for _nova_prefix in ["nova-2/", "nova/"]:
294
+ if _stripped.startswith(_nova_prefix):
295
+ _model_for_id = _model_for_id.replace(_nova_prefix, "", 1)
296
+ break
297
+ modelId = self.encode_model_id(model_id=_model_for_id)
298
+ # Inject region extracted from model path so _get_aws_region_name picks it up
299
+ if (
300
+ _region_from_model is not None
301
+ and "aws_region_name" not in optional_params
302
+ ):
303
+ optional_params["aws_region_name"] = _region_from_model
304
+
305
+ fake_stream = litellm.AmazonConverseConfig().should_fake_stream(
306
+ fake_stream=fake_stream,
307
+ model=model,
308
+ stream=stream,
309
+ custom_llm_provider="bedrock",
310
+ )
311
+
312
+ ### SET REGION NAME ###
313
+ aws_region_name = self._get_aws_region_name(
314
+ optional_params=optional_params,
315
+ model=model,
316
+ model_id=unencoded_model_id,
317
+ )
318
+
319
+ ## CREDENTIALS ##
320
+ # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them
321
+ aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
322
+ aws_access_key_id = optional_params.pop("aws_access_key_id", None)
323
+ aws_session_token = optional_params.pop("aws_session_token", None)
324
+ aws_role_name = optional_params.pop("aws_role_name", None)
325
+ aws_session_name = optional_params.pop("aws_session_name", None)
326
+ aws_profile_name = optional_params.pop("aws_profile_name", None)
327
+ aws_bedrock_runtime_endpoint = optional_params.pop(
328
+ "aws_bedrock_runtime_endpoint", None
329
+ ) # https://bedrock-runtime.{region_name}.amazonaws.com
330
+ aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
331
+ aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
332
+ aws_external_id = optional_params.pop("aws_external_id", None)
333
+ optional_params.pop("aws_region_name", None)
334
+
335
+ litellm_params[
336
+ "aws_region_name"
337
+ ] = aws_region_name # [DO NOT DELETE] important for async calls
338
+
339
+ credentials: Credentials = self.get_credentials(
340
+ aws_access_key_id=aws_access_key_id,
341
+ aws_secret_access_key=aws_secret_access_key,
342
+ aws_session_token=aws_session_token,
343
+ aws_region_name=aws_region_name,
344
+ aws_session_name=aws_session_name,
345
+ aws_profile_name=aws_profile_name,
346
+ aws_role_name=aws_role_name,
347
+ aws_web_identity_token=aws_web_identity_token,
348
+ aws_sts_endpoint=aws_sts_endpoint,
349
+ aws_external_id=aws_external_id,
350
+ )
351
+
352
+ ### SET RUNTIME ENDPOINT ###
353
+ endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
354
+ api_base=api_base,
355
+ aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
356
+ aws_region_name=aws_region_name,
357
+ )
358
+ if (stream is not None and stream is True) and not fake_stream:
359
+ endpoint_url = f"{endpoint_url}/model/{modelId}/converse-stream"
360
+ proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse-stream"
361
+ else:
362
+ endpoint_url = f"{endpoint_url}/model/{modelId}/converse"
363
+ proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse"
364
+
365
+ ## COMPLETION CALL
366
+ headers = {"Content-Type": "application/json"}
367
+ if extra_headers is not None:
368
+ headers = {"Content-Type": "application/json", **extra_headers}
369
+
370
+ # Filter beta headers in HTTP headers before making the request
371
+ headers = update_headers_with_filtered_beta(
372
+ headers=headers, provider="bedrock_converse"
373
+ )
374
+ ### ROUTING (ASYNC, STREAMING, SYNC)
375
+ if acompletion:
376
+ if isinstance(client, HTTPHandler):
377
+ client = None
378
+ if stream is True:
379
+ return self.async_streaming(
380
+ model=model,
381
+ messages=messages,
382
+ api_base=proxy_endpoint_url,
383
+ model_response=model_response,
384
+ encoding=encoding,
385
+ logging_obj=logging_obj,
386
+ optional_params=optional_params,
387
+ stream=True,
388
+ litellm_params=litellm_params,
389
+ logger_fn=logger_fn,
390
+ headers=headers,
391
+ timeout=timeout,
392
+ client=client,
393
+ json_mode=json_mode,
394
+ fake_stream=fake_stream,
395
+ credentials=credentials,
396
+ api_key=api_key,
397
+ stream_chunk_size=stream_chunk_size,
398
+ ) # type: ignore
399
+ ### ASYNC COMPLETION
400
+ return self.async_completion(
401
+ model=model,
402
+ messages=messages,
403
+ api_base=proxy_endpoint_url,
404
+ model_response=model_response,
405
+ encoding=encoding,
406
+ logging_obj=logging_obj,
407
+ optional_params=optional_params,
408
+ stream=stream, # type: ignore
409
+ litellm_params=litellm_params,
410
+ logger_fn=logger_fn,
411
+ headers=headers,
412
+ timeout=timeout,
413
+ client=client,
414
+ credentials=credentials,
415
+ api_key=api_key,
416
+ ) # type: ignore
417
+
418
+ ## TRANSFORMATION ##
419
+
420
+ _data = litellm.AmazonConverseConfig()._transform_request(
421
+ model=model,
422
+ messages=messages,
423
+ optional_params=optional_params,
424
+ litellm_params=litellm_params,
425
+ headers=extra_headers,
426
+ )
427
+ data = json.dumps(_data)
428
+
429
+ prepped = self.get_request_headers(
430
+ credentials=credentials,
431
+ aws_region_name=aws_region_name,
432
+ extra_headers=extra_headers,
433
+ endpoint_url=proxy_endpoint_url,
434
+ data=data,
435
+ headers=headers,
436
+ api_key=api_key,
437
+ )
438
+
439
+ ## LOGGING
440
+ logging_obj.pre_call(
441
+ input=messages,
442
+ api_key="",
443
+ additional_args={
444
+ "complete_input_dict": data,
445
+ "api_base": proxy_endpoint_url,
446
+ "headers": prepped.headers,
447
+ },
448
+ )
449
+ if client is None or isinstance(client, AsyncHTTPHandler):
450
+ _params = {}
451
+ if timeout is not None:
452
+ if isinstance(timeout, float) or isinstance(timeout, int):
453
+ timeout = httpx.Timeout(timeout)
454
+ _params["timeout"] = timeout
455
+ client = _get_httpx_client(_params) # type: ignore
456
+ else:
457
+ client = client
458
+
459
+ if stream is not None and stream is True:
460
+ completion_stream = make_sync_call(
461
+ client=(
462
+ client
463
+ if client is not None and isinstance(client, HTTPHandler)
464
+ else None
465
+ ),
466
+ api_base=proxy_endpoint_url,
467
+ headers=prepped.headers, # type: ignore
468
+ data=data,
469
+ model=model,
470
+ messages=messages,
471
+ logging_obj=logging_obj,
472
+ json_mode=json_mode,
473
+ fake_stream=fake_stream,
474
+ stream_chunk_size=stream_chunk_size,
475
+ )
476
+ streaming_response = CustomStreamWrapper(
477
+ completion_stream=completion_stream,
478
+ model=model,
479
+ custom_llm_provider="bedrock",
480
+ logging_obj=logging_obj,
481
+ )
482
+
483
+ return streaming_response
484
+
485
+ ### COMPLETION
486
+
487
+ try:
488
+ response = client.post(
489
+ url=proxy_endpoint_url,
490
+ headers=prepped.headers,
491
+ data=data,
492
+ logging_obj=logging_obj,
493
+ ) # type: ignore
494
+ response.raise_for_status()
495
+ except httpx.HTTPStatusError as err:
496
+ error_code = err.response.status_code
497
+ raise BedrockError(status_code=error_code, message=err.response.text)
498
+ except httpx.TimeoutException:
499
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
500
+
501
+ return litellm.AmazonConverseConfig()._transform_response(
502
+ model=model,
503
+ response=response,
504
+ model_response=model_response,
505
+ stream=stream if isinstance(stream, bool) else False,
506
+ logging_obj=logging_obj,
507
+ api_key="",
508
+ data=data,
509
+ messages=messages,
510
+ optional_params=optional_params,
511
+ encoding=encoding,
512
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/handler.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ """
2
+ Uses base_llm_http_handler to call the 'converse like' endpoint.
3
+
4
+ Relevant issue: https://github.com/BerriAI/litellm/issues/8085
5
+ """
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/transformation.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ """
2
+ Uses `converse_transformation.py` to transform the messages to the format required by Bedrock Converse.
3
+ """
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_transformation.py ADDED
@@ -0,0 +1,2126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Translating between OpenAI's `/chat/completion` format and Amazon's `/converse` format
3
+ """
4
+
5
+ import copy
6
+ import json
7
+ import time
8
+ import types
9
+ from typing import List, Literal, Optional, Tuple, Union, cast, overload
10
+
11
+ import httpx
12
+
13
+ import litellm
14
+ from litellm._logging import verbose_logger
15
+ from litellm.constants import (
16
+ BEDROCK_MIN_THINKING_BUDGET_TOKENS,
17
+ RESPONSE_FORMAT_TOOL_NAME,
18
+ )
19
+ from litellm.litellm_core_utils.core_helpers import (
20
+ filter_exceptions_from_params,
21
+ filter_internal_params,
22
+ map_finish_reason,
23
+ safe_deep_copy,
24
+ )
25
+ from litellm.litellm_core_utils.litellm_logging import Logging
26
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
27
+ _parse_content_for_reasoning,
28
+ )
29
+ from litellm.litellm_core_utils.prompt_templates.factory import (
30
+ BedrockConverseMessagesProcessor,
31
+ _bedrock_converse_messages_pt,
32
+ _bedrock_tools_pt,
33
+ )
34
+ from litellm.llms.anthropic.chat.transformation import AnthropicConfig
35
+ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
36
+ from litellm.types.llms.bedrock import *
37
+ from litellm.types.llms.openai import (
38
+ AllMessageValues,
39
+ ChatCompletionAssistantMessage,
40
+ ChatCompletionRedactedThinkingBlock,
41
+ ChatCompletionResponseMessage,
42
+ ChatCompletionSystemMessage,
43
+ ChatCompletionThinkingBlock,
44
+ ChatCompletionToolCallChunk,
45
+ ChatCompletionToolCallFunctionChunk,
46
+ ChatCompletionToolParam,
47
+ ChatCompletionToolParamFunctionChunk,
48
+ ChatCompletionUserMessage,
49
+ OpenAIChatCompletionToolParam,
50
+ OpenAIMessageContentListBlock,
51
+ )
52
+ from litellm.types.utils import (
53
+ ChatCompletionMessageToolCall,
54
+ CompletionTokensDetailsWrapper,
55
+ Function,
56
+ Message,
57
+ ModelResponse,
58
+ PromptTokensDetailsWrapper,
59
+ Usage,
60
+ )
61
+ from litellm.utils import (
62
+ add_dummy_tool,
63
+ any_assistant_message_has_thinking_blocks,
64
+ has_tool_call_blocks,
65
+ last_assistant_with_tool_calls_has_no_thinking_blocks,
66
+ supports_reasoning,
67
+ token_counter,
68
+ )
69
+
70
+ from ..common_utils import (
71
+ BedrockError,
72
+ BedrockModelInfo,
73
+ get_anthropic_beta_from_headers,
74
+ get_bedrock_tool_name,
75
+ is_claude_4_5_on_bedrock,
76
+ )
77
+
78
+ # Computer use tool prefixes supported by Bedrock
79
+ BEDROCK_COMPUTER_USE_TOOLS = [
80
+ "computer_use_preview",
81
+ "computer_",
82
+ "bash_",
83
+ "text_editor_",
84
+ ]
85
+
86
+ # Beta header patterns that are not supported by Bedrock Converse API
87
+ # These will be filtered out to prevent errors
88
+ UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS = [
89
+ "advanced-tool-use", # Bedrock Converse doesn't support advanced-tool-use beta headers
90
+ "prompt-caching", # Prompt caching not supported in Converse API
91
+ "compact-2026-01-12", # The compact beta feature is not currently supported on the Converse and ConverseStream APIs
92
+ ]
93
+
94
+ # Models that support Bedrock's native structured outputs API (outputConfig.textFormat)
95
+ # Uses substring matching against the Bedrock model ID
96
+ # Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
97
+ BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS = {
98
+ # Anthropic Claude 4.5+
99
+ "claude-haiku-4-5",
100
+ "claude-sonnet-4-5",
101
+ "claude-opus-4-5",
102
+ "claude-opus-4-6",
103
+ # Qwen3
104
+ "qwen3",
105
+ # DeepSeek
106
+ "deepseek-v3.1",
107
+ # Gemma 3
108
+ "gemma-3",
109
+ # MiniMax
110
+ "minimax-m2",
111
+ # Mistral (magistral-small excluded: broken constrained decoding on Bedrock)
112
+ "ministral",
113
+ "mistral-large-3",
114
+ "voxtral",
115
+ # Moonshot
116
+ "kimi-k2",
117
+ # NVIDIA
118
+ "nemotron-nano",
119
+ # OpenAI (gpt-oss excluded: broken constrained decoding, works via tool-call fallback)
120
+ }
121
+
122
+
123
+ class AmazonConverseConfig(BaseConfig):
124
+ """
125
+ Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
126
+ #2 - https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html#conversation-inference-supported-models-features
127
+ """
128
+
129
+ maxTokens: Optional[int]
130
+ stopSequences: Optional[List[str]]
131
+ temperature: Optional[int]
132
+ topP: Optional[int]
133
+ topK: Optional[int]
134
+
135
+ def __init__(
136
+ self,
137
+ maxTokens: Optional[int] = None,
138
+ stopSequences: Optional[List[str]] = None,
139
+ temperature: Optional[int] = None,
140
+ topP: Optional[int] = None,
141
+ topK: Optional[int] = None,
142
+ ) -> None:
143
+ locals_ = locals().copy()
144
+ for key, value in locals_.items():
145
+ if key != "self" and value is not None:
146
+ setattr(self.__class__, key, value)
147
+
148
+ @property
149
+ def custom_llm_provider(self) -> Optional[str]:
150
+ return "bedrock_converse"
151
+
152
+ @classmethod
153
+ def get_config_blocks(cls) -> dict:
154
+ return {
155
+ "guardrailConfig": GuardrailConfigBlock,
156
+ "performanceConfig": PerformanceConfigBlock,
157
+ "serviceTier": ServiceTierBlock,
158
+ }
159
+
160
+ @staticmethod
161
+ def _convert_consecutive_user_messages_to_guarded_text(
162
+ messages: List[AllMessageValues], optional_params: dict
163
+ ) -> List[AllMessageValues]:
164
+ """
165
+ Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present
166
+ and no guarded_text is already present in those messages.
167
+ """
168
+ # Check if guardrailConfig is present
169
+ if "guardrailConfig" not in optional_params:
170
+ return messages
171
+
172
+ # Find all consecutive user messages at the end
173
+ consecutive_user_message_indices = []
174
+ for i in range(len(messages) - 1, -1, -1):
175
+ if messages[i].get("role") == "user":
176
+ consecutive_user_message_indices.append(i)
177
+ else:
178
+ break
179
+
180
+ if not consecutive_user_message_indices:
181
+ return messages
182
+
183
+ # Process each consecutive user message
184
+ messages_copy = copy.deepcopy(messages)
185
+ for user_message_index in consecutive_user_message_indices:
186
+ user_message = messages_copy[user_message_index]
187
+ content = user_message.get("content", [])
188
+
189
+ if isinstance(content, list):
190
+ has_guarded_text = any(
191
+ isinstance(item, dict) and item.get("type") == "guarded_text"
192
+ for item in content
193
+ )
194
+ if has_guarded_text:
195
+ continue # Skip this message if it already has guarded_text
196
+
197
+ # Convert text elements to guarded_text
198
+ new_content = []
199
+ for item in content:
200
+ if isinstance(item, dict) and item.get("type") == "text":
201
+ new_item = {"type": "guarded_text", "text": item["text"]} # type: ignore
202
+ new_content.append(new_item)
203
+ else:
204
+ new_content.append(item)
205
+
206
+ messages_copy[user_message_index]["content"] = new_content # type: ignore
207
+ elif isinstance(content, str):
208
+ # If content is a string, convert it to guarded_text
209
+ messages_copy[user_message_index]["content"] = [ # type: ignore
210
+ {"type": "guarded_text", "text": content} # type: ignore
211
+ ]
212
+
213
+ return messages_copy
214
+
215
+ @classmethod
216
+ def get_config(cls):
217
+ return {
218
+ k: v
219
+ for k, v in cls.__dict__.items()
220
+ if not k.startswith("__")
221
+ and not isinstance(
222
+ v,
223
+ (
224
+ types.FunctionType,
225
+ types.BuiltinFunctionType,
226
+ classmethod,
227
+ staticmethod,
228
+ ),
229
+ )
230
+ and v is not None
231
+ }
232
+
233
+ def _validate_request_metadata(self, metadata: dict) -> None:
234
+ """
235
+ Validate requestMetadata according to AWS Bedrock Converse API constraints.
236
+
237
+ Constraints:
238
+ - Maximum of 16 items
239
+ - Keys: 1-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{1,256}
240
+ - Values: 0-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{0,256}
241
+ """
242
+ import re
243
+
244
+ if not isinstance(metadata, dict):
245
+ raise litellm.exceptions.BadRequestError(
246
+ message="requestMetadata must be a dictionary",
247
+ model="bedrock",
248
+ llm_provider="bedrock",
249
+ )
250
+
251
+ if len(metadata) > 16:
252
+ raise litellm.exceptions.BadRequestError(
253
+ message="requestMetadata can contain a maximum of 16 items",
254
+ model="bedrock",
255
+ llm_provider="bedrock",
256
+ )
257
+
258
+ key_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
259
+ value_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
260
+
261
+ for key, value in metadata.items():
262
+ if not isinstance(key, str):
263
+ raise litellm.exceptions.BadRequestError(
264
+ message="requestMetadata keys must be strings",
265
+ model="bedrock",
266
+ llm_provider="bedrock",
267
+ )
268
+
269
+ if not isinstance(value, str):
270
+ raise litellm.exceptions.BadRequestError(
271
+ message="requestMetadata values must be strings",
272
+ model="bedrock",
273
+ llm_provider="bedrock",
274
+ )
275
+
276
+ if len(key) == 0 or len(key) > 256:
277
+ raise litellm.exceptions.BadRequestError(
278
+ message="requestMetadata key length must be 1-256 characters",
279
+ model="bedrock",
280
+ llm_provider="bedrock",
281
+ )
282
+
283
+ if len(value) > 256:
284
+ raise litellm.exceptions.BadRequestError(
285
+ message="requestMetadata value length must be 0-256 characters",
286
+ model="bedrock",
287
+ llm_provider="bedrock",
288
+ )
289
+
290
+ if not key_pattern.match(key):
291
+ raise litellm.exceptions.BadRequestError(
292
+ message=f"requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
293
+ model="bedrock",
294
+ llm_provider="bedrock",
295
+ )
296
+
297
+ if not value_pattern.match(value):
298
+ raise litellm.exceptions.BadRequestError(
299
+ message=f"requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
300
+ model="bedrock",
301
+ llm_provider="bedrock",
302
+ )
303
+
304
+ def _is_nova_2_model(self, model: str) -> bool:
305
+ """
306
+ Check if the model is a Nova 2 model that supports reasoningConfig.
307
+
308
+ Nova 2 models use a different reasoning configuration structure compared to
309
+ Anthropic's thinking parameter and GPT-OSS's reasoning_effort parameter.
310
+
311
+ Supported models:
312
+ - amazon.nova-2-lite-v1:0
313
+ - amazon.nova-2-pro-preview-20251202-v1:0
314
+ - us.amazon.nova-2-lite-v1:0
315
+ - eu.amazon.nova-2-lite-v1:0
316
+ - apac.amazon.nova-2-lite-v1:0
317
+ - (and other regional variants)
318
+
319
+ Args:
320
+ model: The model identifier
321
+
322
+ Returns:
323
+ True if the model is a Nova 2 model, False otherwise
324
+
325
+ Examples:
326
+ >>> config = AmazonConverseConfig()
327
+ >>> config._is_nova_2_model("amazon.nova-2-lite-v1:0")
328
+ True
329
+ >>> config._is_nova_2_model("us.amazon.nova-2-lite-v1:0")
330
+ True
331
+ >>> config._is_nova_2_model("us.amazon.nova-2-pro-preview-20251202-v1:0")
332
+ True
333
+ >>> config._is_nova_2_model("amazon.nova-pro-1-5-v1:0")
334
+ False
335
+ >>> config._is_nova_2_model("amazon.nova-pro-v1:0")
336
+ False
337
+ """
338
+ # Remove provider routing prefix if present (bedrock/converse/, bedrock/, converse/)
339
+ model_without_region = model
340
+ for routing_prefix in ["bedrock/converse/", "bedrock/", "converse/"]:
341
+ if model_without_region.startswith(routing_prefix):
342
+ model_without_region = model_without_region[len(routing_prefix) :]
343
+ break
344
+
345
+ # Remove regional prefix if present (us., eu., apac.)
346
+ for prefix in ["us.", "eu.", "apac."]:
347
+ if model_without_region.startswith(prefix):
348
+ model_without_region = model_without_region[len(prefix) :]
349
+ break
350
+
351
+ # Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.)
352
+ # Also check for nova-2/ spec prefix for imported models
353
+ return model_without_region.startswith(
354
+ "amazon.nova-2-"
355
+ ) or model_without_region.startswith("nova-2/")
356
+
357
+ def _map_web_search_options(
358
+ self, web_search_options: dict, model: str
359
+ ) -> Optional[BedrockToolBlock]:
360
+ """
361
+ Map web_search_options to Nova grounding systemTool.
362
+
363
+ Nova grounding (web search) is only supported on Amazon Nova models.
364
+ Returns None for non-Nova models.
365
+
366
+ Args:
367
+ web_search_options: The web_search_options dict from the request
368
+ model: The model identifier string
369
+
370
+ Returns:
371
+ BedrockToolBlock with systemTool for Nova models, None otherwise
372
+
373
+ Reference: https://docs.aws.amazon.com/nova/latest/userguide/grounding.html
374
+ """
375
+ # Only Nova models support nova_grounding
376
+ # Model strings can be like: "amazon.nova-pro-v1:0", "us.amazon.nova-pro-v1:0", etc.
377
+ if "nova" not in model.lower():
378
+ verbose_logger.debug(
379
+ f"web_search_options passed but model {model} is not a Nova model. "
380
+ "Nova grounding is only supported on Amazon Nova models."
381
+ )
382
+ return None
383
+
384
+ # Nova doesn't support search_context_size or user_location params
385
+ # (unlike Anthropic), so we just enable grounding with no options
386
+ return BedrockToolBlock(systemTool={"name": "nova_grounding"})
387
+
388
+ def _transform_reasoning_effort_to_reasoning_config(
389
+ self, reasoning_effort: str
390
+ ) -> dict:
391
+ """
392
+ Transform reasoning_effort parameter to Nova 2 reasoningConfig structure.
393
+
394
+ Nova 2 models use a reasoningConfig structure in additionalModelRequestFields
395
+ that differs from both Anthropic's thinking parameter and GPT-OSS's reasoning_effort.
396
+
397
+ Args:
398
+ reasoning_effort: The reasoning effort level, must be "low" or "high"
399
+
400
+ Returns:
401
+ dict: A dictionary containing the reasoningConfig structure:
402
+ {
403
+ "reasoningConfig": {
404
+ "type": "enabled",
405
+ "maxReasoningEffort": "low" | "medium" |"high"
406
+ }
407
+ }
408
+
409
+ Raises:
410
+ BadRequestError: If reasoning_effort is not "low", "medium" or "high"
411
+
412
+ Examples:
413
+ >>> config = AmazonConverseConfig()
414
+ >>> config._transform_reasoning_effort_to_reasoning_config("high")
415
+ {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}}
416
+ >>> config._transform_reasoning_effort_to_reasoning_config("low")
417
+ {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'low'}}
418
+ """
419
+ valid_values = ["low", "medium", "high"]
420
+ if reasoning_effort not in valid_values:
421
+ raise litellm.exceptions.BadRequestError(
422
+ message=f"Invalid reasoning_effort value '{reasoning_effort}' for Nova 2 models. "
423
+ f"Supported values: {valid_values}",
424
+ model="amazon.nova-2-lite-v1:0",
425
+ llm_provider="bedrock_converse",
426
+ )
427
+
428
+ return {
429
+ "reasoningConfig": {
430
+ "type": "enabled",
431
+ "maxReasoningEffort": reasoning_effort,
432
+ }
433
+ }
434
+
435
+ def _handle_reasoning_effort_parameter(
436
+ self, model: str, reasoning_effort: str, optional_params: dict
437
+ ) -> None:
438
+ """
439
+ Handle the reasoning_effort parameter based on the model type.
440
+
441
+ Different model families handle reasoning effort differently:
442
+ - GPT-OSS models: Keep reasoning_effort as-is (passed to additionalModelRequestFields)
443
+ - Nova 2 models: Transform to reasoningConfig structure
444
+ - Other models (Anthropic, etc.): Convert to thinking parameter
445
+
446
+ Args:
447
+ model: The model identifier
448
+ reasoning_effort: The reasoning effort value
449
+ optional_params: Dictionary of optional parameters to update in-place
450
+
451
+ Examples:
452
+ >>> config = AmazonConverseConfig()
453
+ >>> params = {}
454
+ >>> config._handle_reasoning_effort_parameter("gpt-oss-model", "high", params)
455
+ >>> params
456
+ {'reasoning_effort': 'high'}
457
+
458
+ >>> params = {}
459
+ >>> config._handle_reasoning_effort_parameter("amazon.nova-2-lite-v1:0", "high", params)
460
+ >>> params
461
+ {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}}
462
+
463
+ >>> params = {}
464
+ >>> config._handle_reasoning_effort_parameter("anthropic.claude-3", "high", params)
465
+ >>> params
466
+ {'thinking': {'type': 'enabled', 'budget_tokens': 10000}}
467
+ """
468
+ if "gpt-oss" in model:
469
+ # GPT-OSS models: keep reasoning_effort as-is
470
+ # It will be passed through to additionalModelRequestFields
471
+ optional_params["reasoning_effort"] = reasoning_effort
472
+ elif self._is_nova_2_model(model):
473
+ # Nova 2 models: transform to reasoningConfig
474
+ reasoning_config = self._transform_reasoning_effort_to_reasoning_config(
475
+ reasoning_effort
476
+ )
477
+ optional_params.update(reasoning_config)
478
+ else:
479
+ # Anthropic and other models: convert to thinking parameter
480
+ optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
481
+ reasoning_effort=reasoning_effort, model=model
482
+ )
483
+
484
+ @staticmethod
485
+ def _clamp_thinking_budget_tokens(optional_params: dict) -> None:
486
+ """
487
+ Clamp thinking.budget_tokens to the Bedrock minimum (1024).
488
+
489
+ Bedrock returns a 400 error if budget_tokens < 1024.
490
+ """
491
+ thinking = optional_params.get("thinking")
492
+ if isinstance(thinking, dict):
493
+ budget = thinking.get("budget_tokens")
494
+ if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS:
495
+ verbose_logger.debug(
496
+ "Bedrock requires thinking.budget_tokens >= %d, got %d. "
497
+ "Clamping to minimum.",
498
+ BEDROCK_MIN_THINKING_BUDGET_TOKENS,
499
+ budget,
500
+ )
501
+ thinking["budget_tokens"] = BEDROCK_MIN_THINKING_BUDGET_TOKENS
502
+
503
+ def get_supported_openai_params(self, model: str) -> List[str]:
504
+ from litellm.utils import supports_function_calling
505
+
506
+ supported_params = [
507
+ "max_tokens",
508
+ "max_completion_tokens",
509
+ "stream",
510
+ "stream_options",
511
+ "stop",
512
+ "temperature",
513
+ "top_p",
514
+ "extra_headers",
515
+ "response_format",
516
+ "requestMetadata",
517
+ "service_tier",
518
+ "parallel_tool_calls",
519
+ ]
520
+
521
+ if (
522
+ "arn" in model
523
+ ): # we can't infer the model from the arn, so just add all params
524
+ supported_params.append("tools")
525
+ supported_params.append("tool_choice")
526
+ supported_params.append("thinking")
527
+ supported_params.append("reasoning_effort")
528
+ # For nova imported models, also add web_search_options
529
+ if "nova" in model.lower():
530
+ supported_params.append("web_search_options")
531
+ return supported_params
532
+
533
+ ## Filter out 'cross-region' from model name
534
+ base_model = BedrockModelInfo.get_base_model(model)
535
+
536
+ if (
537
+ base_model.startswith("anthropic")
538
+ or base_model.startswith("mistral")
539
+ or base_model.startswith("cohere")
540
+ or base_model.startswith("meta.llama3-1")
541
+ or base_model.startswith("meta.llama3-2")
542
+ or base_model.startswith("meta.llama3-3")
543
+ or base_model.startswith("meta.llama4")
544
+ or base_model.startswith("amazon.nova")
545
+ or supports_function_calling(
546
+ model=model, custom_llm_provider=self.custom_llm_provider
547
+ )
548
+ ):
549
+ supported_params.append("tools")
550
+
551
+ # Nova models support web_search_options (mapped to nova_grounding systemTool)
552
+ if base_model.startswith("amazon.nova"):
553
+ supported_params.append("web_search_options")
554
+
555
+ if litellm.utils.supports_tool_choice(
556
+ model=model, custom_llm_provider=self.custom_llm_provider
557
+ ) or litellm.utils.supports_tool_choice(
558
+ model=base_model, custom_llm_provider=self.custom_llm_provider
559
+ ):
560
+ # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
561
+ supported_params.append("tool_choice")
562
+
563
+ if "gpt-oss" in model:
564
+ supported_params.append("reasoning_effort")
565
+ elif self._is_nova_2_model(model):
566
+ # Nova 2 models support reasoning_effort (transformed to reasoningConfig)
567
+ # These models use a different reasoning structure than Anthropic's thinking parameter
568
+ supported_params.append("reasoning_effort")
569
+ elif (
570
+ "claude-3-7" in model
571
+ or "claude-sonnet-4" in model
572
+ or "claude-opus-4" in model
573
+ or "deepseek.r1" in model
574
+ or supports_reasoning(
575
+ model=model,
576
+ custom_llm_provider=self.custom_llm_provider,
577
+ )
578
+ or supports_reasoning(
579
+ model=base_model, custom_llm_provider=self.custom_llm_provider
580
+ )
581
+ ):
582
+ supported_params.append("thinking")
583
+ supported_params.append("reasoning_effort")
584
+ return supported_params
585
+
586
+ def map_tool_choice_values(
587
+ self, model: str, tool_choice: Union[str, dict], drop_params: bool
588
+ ) -> Optional[ToolChoiceValuesBlock]:
589
+ if tool_choice == "none":
590
+ if litellm.drop_params is True or drop_params is True:
591
+ return None
592
+ else:
593
+ raise litellm.utils.UnsupportedParamsError(
594
+ message="Bedrock doesn't support tool_choice={}. To drop it from the call, set `litellm.drop_params = True.".format(
595
+ tool_choice
596
+ ),
597
+ status_code=400,
598
+ )
599
+ elif tool_choice == "required":
600
+ return ToolChoiceValuesBlock(any={})
601
+ elif tool_choice == "auto":
602
+ return ToolChoiceValuesBlock(auto={})
603
+ elif isinstance(tool_choice, dict):
604
+ # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
605
+ specific_tool = SpecificToolChoiceBlock(
606
+ name=tool_choice.get("function", {}).get("name", "")
607
+ )
608
+ return ToolChoiceValuesBlock(tool=specific_tool)
609
+ else:
610
+ raise litellm.utils.UnsupportedParamsError(
611
+ message="Bedrock doesn't support tool_choice={}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.".format(
612
+ tool_choice
613
+ ),
614
+ status_code=400,
615
+ )
616
+
617
+ def get_supported_image_types(self) -> List[str]:
618
+ return ["png", "jpeg", "gif", "webp"]
619
+
620
+ def get_supported_document_types(self) -> List[str]:
621
+ return ["pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"]
622
+
623
+ def get_supported_video_types(self) -> List[str]:
624
+ return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"]
625
+
626
+ def get_all_supported_content_types(self) -> List[str]:
627
+ return (
628
+ self.get_supported_image_types()
629
+ + self.get_supported_document_types()
630
+ + self.get_supported_video_types()
631
+ )
632
+
633
+ def is_computer_use_tool_used(
634
+ self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str
635
+ ) -> bool:
636
+ """Check if computer use tools are being used in the request."""
637
+ if tools is None:
638
+ return False
639
+
640
+ for tool in tools:
641
+ if "type" in tool:
642
+ tool_type = tool["type"]
643
+ for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS:
644
+ if tool_type.startswith(computer_use_prefix):
645
+ return True
646
+ return False
647
+
648
+ def _transform_computer_use_tools(
649
+ self, computer_use_tools: List[OpenAIChatCompletionToolParam]
650
+ ) -> List[dict]:
651
+ """Transform computer use tools to Bedrock format."""
652
+ transformed_tools: List[dict] = []
653
+
654
+ for tool in computer_use_tools:
655
+ tool_type = tool.get("type", "")
656
+
657
+ # Check if this is a computer use tool with the startswith method
658
+ is_computer_use_tool = False
659
+ for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS:
660
+ if tool_type.startswith(computer_use_prefix):
661
+ is_computer_use_tool = True
662
+ break
663
+
664
+ transformed_tool: dict = {}
665
+ if is_computer_use_tool:
666
+ if tool_type.startswith("computer_") and "function" in tool:
667
+ # Computer use tool with function format
668
+ func = tool["function"]
669
+ transformed_tool = {
670
+ "type": tool_type,
671
+ "name": func.get("name", "computer"),
672
+ **func.get("parameters", {}),
673
+ }
674
+ else:
675
+ # Direct tools - just need to ensure name is present
676
+ transformed_tool = dict(tool)
677
+ if "name" not in transformed_tool:
678
+ if tool_type.startswith("bash_"):
679
+ transformed_tool["name"] = "bash"
680
+ elif tool_type.startswith("text_editor_"):
681
+ transformed_tool["name"] = "str_replace_editor"
682
+ else:
683
+ # Pass through other tools as-is
684
+ transformed_tool = dict(tool)
685
+
686
+ transformed_tools.append(transformed_tool)
687
+
688
+ return transformed_tools
689
+
690
+ def _separate_computer_use_tools(
691
+ self, tools: List[OpenAIChatCompletionToolParam], model: str
692
+ ) -> Tuple[
693
+ List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]
694
+ ]:
695
+ """
696
+ Separate computer use tools from regular function tools.
697
+
698
+ Args:
699
+ tools: List of tools to separate
700
+ model: The model name to check if it supports computer use
701
+
702
+ Returns:
703
+ Tuple of (computer_use_tools, regular_tools)
704
+ """
705
+ computer_use_tools = []
706
+ regular_tools = []
707
+
708
+ for tool in tools:
709
+ if "type" in tool:
710
+ tool_type = tool["type"]
711
+ is_computer_use_tool = False
712
+ for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS:
713
+ if tool_type.startswith(computer_use_prefix):
714
+ is_computer_use_tool = True
715
+ break
716
+ if is_computer_use_tool:
717
+ computer_use_tools.append(tool)
718
+ else:
719
+ regular_tools.append(tool)
720
+ else:
721
+ regular_tools.append(tool)
722
+
723
+ return computer_use_tools, regular_tools
724
+
725
+ def _create_json_tool_call_for_response_format(
726
+ self,
727
+ json_schema: Optional[dict] = None,
728
+ description: Optional[str] = None,
729
+ ) -> ChatCompletionToolParam:
730
+ """
731
+ Handles creating a tool call for getting responses in JSON format.
732
+
733
+ Args:
734
+ json_schema (Optional[dict]): The JSON schema the response should be in
735
+
736
+ Returns:
737
+ AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format
738
+ """
739
+
740
+ if json_schema is None:
741
+ # Anthropic raises a 400 BadRequest error if properties is passed as None
742
+ # see usage with additionalProperties (Example 5) https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb
743
+ _input_schema = {
744
+ "type": "object",
745
+ "additionalProperties": True,
746
+ "properties": {},
747
+ }
748
+ else:
749
+ # Use the schema as-is for Bedrock
750
+ # Bedrock requires the tool schema to be of type "object" and doesn't need unwrapping
751
+ _input_schema = json_schema
752
+
753
+ tool_param_function_chunk = ChatCompletionToolParamFunctionChunk(
754
+ name=RESPONSE_FORMAT_TOOL_NAME, parameters=_input_schema
755
+ )
756
+ if description:
757
+ tool_param_function_chunk["description"] = description
758
+
759
+ _tool = ChatCompletionToolParam(
760
+ type="function",
761
+ function=tool_param_function_chunk,
762
+ )
763
+ return _tool
764
+
765
+ @staticmethod
766
+ def _supports_native_structured_outputs(model: str) -> bool:
767
+ """Check if the Bedrock model supports native structured outputs (outputConfig.textFormat)."""
768
+ return any(
769
+ substring in model for substring in BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS
770
+ )
771
+
772
+ @staticmethod
773
+ def _add_additional_properties_to_schema(schema: dict) -> dict:
774
+ """
775
+ Recursively ensure all object types in a JSON schema have
776
+ ``"additionalProperties": false``.
777
+
778
+ Bedrock's native structured-outputs API requires this field to be
779
+ explicitly set on every object node, otherwise it returns a
780
+ validation error.
781
+ """
782
+ if not isinstance(schema, dict):
783
+ return schema
784
+
785
+ result = dict(schema)
786
+
787
+ if result.get("type") == "object" and "additionalProperties" not in result:
788
+ result["additionalProperties"] = False
789
+
790
+ # Recurse into nested schemas
791
+ if "properties" in result and isinstance(result["properties"], dict):
792
+ result["properties"] = {
793
+ k: AmazonConverseConfig._add_additional_properties_to_schema(v)
794
+ for k, v in result["properties"].items()
795
+ }
796
+ if "items" in result and isinstance(result["items"], dict):
797
+ result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(
798
+ result["items"]
799
+ )
800
+ for defs_key in ("$defs", "definitions"):
801
+ if defs_key in result and isinstance(result[defs_key], dict):
802
+ result[defs_key] = {
803
+ k: AmazonConverseConfig._add_additional_properties_to_schema(v)
804
+ for k, v in result[defs_key].items()
805
+ }
806
+ for key in ("anyOf", "allOf", "oneOf"):
807
+ if key in result and isinstance(result[key], list):
808
+ result[key] = [
809
+ AmazonConverseConfig._add_additional_properties_to_schema(item)
810
+ for item in result[key]
811
+ ]
812
+
813
+ return result
814
+
815
+ @staticmethod
816
+ def _create_output_config_for_response_format(
817
+ json_schema: Optional[dict] = None,
818
+ name: Optional[str] = None,
819
+ description: Optional[str] = None,
820
+ ) -> "OutputConfigBlock":
821
+ """
822
+ Build an outputConfig block for Bedrock's native structured outputs API.
823
+
824
+ The Converse API expects:
825
+ {
826
+ "outputConfig": {
827
+ "textFormat": {
828
+ "type": "json_schema",
829
+ "structure": {
830
+ "jsonSchema": {
831
+ "schema": "<json-string>",
832
+ "name": "optional",
833
+ "description": "optional"
834
+ }
835
+ }
836
+ }
837
+ }
838
+ }
839
+ """
840
+ if json_schema is not None:
841
+ json_schema = AmazonConverseConfig._add_additional_properties_to_schema(
842
+ json_schema
843
+ )
844
+ schema_str = json.dumps(json_schema) if json_schema is not None else "{}"
845
+ json_schema_def: JsonSchemaDefinition = {"schema": schema_str}
846
+ if name is not None:
847
+ json_schema_def["name"] = name
848
+ if description is not None:
849
+ json_schema_def["description"] = description
850
+
851
+ return OutputConfigBlock(
852
+ textFormat=OutputFormat(
853
+ type="json_schema",
854
+ structure=OutputFormatStructure(jsonSchema=json_schema_def),
855
+ )
856
+ )
857
+
858
+ def _apply_tool_call_transformation(
859
+ self,
860
+ tools: List[OpenAIChatCompletionToolParam],
861
+ model: str,
862
+ non_default_params: dict,
863
+ optional_params: dict,
864
+ ):
865
+ optional_params = self._add_tools_to_optional_params(
866
+ optional_params=optional_params, tools=tools
867
+ )
868
+
869
+ if (
870
+ "meta.llama3-3-70b-instruct-v1:0" in model
871
+ and non_default_params.get("stream", False) is True
872
+ ):
873
+ optional_params["fake_stream"] = True
874
+
875
+ def map_openai_params(
876
+ self,
877
+ non_default_params: dict,
878
+ optional_params: dict,
879
+ model: str,
880
+ drop_params: bool,
881
+ ) -> dict:
882
+ is_thinking_enabled = self.is_thinking_enabled(non_default_params)
883
+
884
+ for param, value in non_default_params.items():
885
+ if param == "response_format" and isinstance(value, dict):
886
+ optional_params = self._translate_response_format_param(
887
+ value=value,
888
+ model=model,
889
+ optional_params=optional_params,
890
+ non_default_params=non_default_params,
891
+ is_thinking_enabled=is_thinking_enabled,
892
+ )
893
+ if param == "max_tokens" or param == "max_completion_tokens":
894
+ optional_params["maxTokens"] = value
895
+ if param == "stream":
896
+ optional_params["stream"] = value
897
+ if param == "stop":
898
+ if isinstance(value, str):
899
+ if len(value) == 0: # converse raises error for empty strings
900
+ continue
901
+ value = [value]
902
+ optional_params["stopSequences"] = value
903
+ if param == "temperature":
904
+ optional_params["temperature"] = value
905
+ if param == "top_p":
906
+ optional_params["topP"] = value
907
+ if param == "tools" and isinstance(value, list):
908
+ self._apply_tool_call_transformation(
909
+ tools=cast(List[OpenAIChatCompletionToolParam], value),
910
+ model=model,
911
+ non_default_params=non_default_params,
912
+ optional_params=optional_params,
913
+ )
914
+ if param == "tool_choice":
915
+ _tool_choice_value = self.map_tool_choice_values(
916
+ model=model, tool_choice=value, drop_params=drop_params # type: ignore
917
+ )
918
+ if _tool_choice_value is not None:
919
+ optional_params["tool_choice"] = _tool_choice_value
920
+ if param == "parallel_tool_calls":
921
+ disable_parallel = not value
922
+ optional_params["_parallel_tool_use_config"] = {
923
+ "tool_choice": {"disable_parallel_tool_use": disable_parallel}
924
+ }
925
+ if param == "thinking":
926
+ optional_params["thinking"] = value
927
+ elif param == "reasoning_effort" and isinstance(value, str):
928
+ self._handle_reasoning_effort_parameter(
929
+ model=model, reasoning_effort=value, optional_params=optional_params
930
+ )
931
+ if param == "requestMetadata":
932
+ if value is not None and isinstance(value, dict):
933
+ self._validate_request_metadata(value) # type: ignore
934
+ optional_params["requestMetadata"] = value
935
+ if param == "service_tier" and isinstance(value, str):
936
+ self._map_service_tier_param(value, optional_params)
937
+
938
+ if param == "web_search_options" and isinstance(value, dict):
939
+ # Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
940
+ # because empty dict {} is falsy but is a valid way to enable Nova grounding
941
+ grounding_tool = self._map_web_search_options(value, model)
942
+ if grounding_tool is not None:
943
+ optional_params = self._add_tools_to_optional_params(
944
+ optional_params=optional_params, tools=[grounding_tool]
945
+ )
946
+
947
+ # Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
948
+ # Nova 2 handles token budgeting differently through reasoningConfig
949
+ if "gpt-oss" not in model and not self._is_nova_2_model(model):
950
+ self.update_optional_params_with_thinking_tokens(
951
+ non_default_params=non_default_params, optional_params=optional_params
952
+ )
953
+
954
+ final_is_thinking_enabled = self.is_thinking_enabled(optional_params)
955
+ if final_is_thinking_enabled and "tool_choice" in optional_params:
956
+ tool_choice_block = optional_params["tool_choice"]
957
+ if isinstance(tool_choice_block, dict):
958
+ if "any" in tool_choice_block or "tool" in tool_choice_block:
959
+ verbose_logger.info(
960
+ f"{model} does not support forced tool use (tool_choice='required' or specific tool) "
961
+ f"when reasoning is enabled. Changing tool_choice to 'auto'."
962
+ )
963
+ optional_params["tool_choice"] = ToolChoiceValuesBlock(auto={})
964
+
965
+ return optional_params
966
+
967
+ def _map_service_tier_param(self, value: str, optional_params: dict) -> None:
968
+ """Map OpenAI service_tier (string) to Bedrock serviceTier (object).
969
+
970
+ OpenAI values: "auto", "default", "flex", "priority"
971
+ Bedrock values: "default", "flex", "priority" (no "auto")
972
+ """
973
+ bedrock_tier = value
974
+ if value == "auto":
975
+ bedrock_tier = "default" # Bedrock doesn't support "auto"
976
+ if bedrock_tier in ("default", "flex", "priority"):
977
+ optional_params["serviceTier"] = {"type": bedrock_tier}
978
+
979
+ def _translate_response_format_param(
980
+ self,
981
+ value: dict,
982
+ model: str,
983
+ optional_params: dict,
984
+ non_default_params: dict,
985
+ is_thinking_enabled: bool,
986
+ ) -> dict:
987
+ """
988
+ Handles translation of response_format parameter to Bedrock format.
989
+
990
+ Returns `optional_params` with the translated response_format parameter.
991
+ """
992
+ ignore_response_format_types = ["text"]
993
+ if value["type"] in ignore_response_format_types: # value is a no-op
994
+ return optional_params
995
+
996
+ json_schema: Optional[dict] = None
997
+ name: Optional[str] = None
998
+ description: Optional[str] = None
999
+ if "response_schema" in value:
1000
+ json_schema = value["response_schema"]
1001
+ elif "json_schema" in value:
1002
+ json_schema = value["json_schema"]["schema"]
1003
+ name = value["json_schema"].get("name")
1004
+ description = value["json_schema"].get("description")
1005
+
1006
+ if "type" in value and value["type"] == "text":
1007
+ return optional_params
1008
+
1009
+ if self._supports_native_structured_outputs(model) and json_schema is not None:
1010
+ # Use Bedrock's native structured outputs API (outputConfig.textFormat)
1011
+ # No synthetic tool injection, no fake_stream needed.
1012
+ # Requires an explicit schema — json_object with no schema falls through
1013
+ # to the tool-call path below.
1014
+ output_config = self._create_output_config_for_response_format(
1015
+ json_schema=json_schema,
1016
+ name=name,
1017
+ description=description,
1018
+ )
1019
+ optional_params["outputConfig"] = output_config
1020
+ else:
1021
+ # Fallback: translate to a synthetic tool call
1022
+ # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
1023
+ _tool = self._create_json_tool_call_for_response_format(
1024
+ json_schema=json_schema,
1025
+ description=description,
1026
+ )
1027
+ optional_params = self._add_tools_to_optional_params(
1028
+ optional_params=optional_params, tools=[_tool]
1029
+ )
1030
+
1031
+ if (
1032
+ litellm.utils.supports_tool_choice(
1033
+ model=model, custom_llm_provider=self.custom_llm_provider
1034
+ )
1035
+ and not is_thinking_enabled
1036
+ ):
1037
+ optional_params["tool_choice"] = ToolChoiceValuesBlock(
1038
+ tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
1039
+ )
1040
+ if non_default_params.get("stream", False) is True:
1041
+ optional_params["fake_stream"] = True
1042
+
1043
+ optional_params["json_mode"] = True
1044
+ return optional_params
1045
+
1046
+ def update_optional_params_with_thinking_tokens(
1047
+ self, non_default_params: dict, optional_params: dict
1048
+ ):
1049
+ """
1050
+ Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.
1051
+
1052
+ Checks 'non_default_params' for 'thinking' and 'max_tokens'
1053
+
1054
+ if 'thinking' is enabled and 'max_tokens' is not specified, set 'max_tokens' to the thinking token budget + DEFAULT_MAX_TOKENS
1055
+
1056
+ Also clamps thinking.budget_tokens to the Bedrock minimum (1024) to
1057
+ prevent 400 errors from the Bedrock API.
1058
+ """
1059
+ from litellm.constants import DEFAULT_MAX_TOKENS
1060
+
1061
+ self._clamp_thinking_budget_tokens(optional_params)
1062
+
1063
+ is_thinking_enabled = self.is_thinking_enabled(optional_params)
1064
+ is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params)
1065
+ if is_thinking_enabled and not is_max_tokens_in_request:
1066
+ thinking_token_budget = cast(dict, optional_params["thinking"]).get(
1067
+ "budget_tokens", None
1068
+ )
1069
+ if thinking_token_budget is not None:
1070
+ optional_params["maxTokens"] = (
1071
+ thinking_token_budget + DEFAULT_MAX_TOKENS
1072
+ )
1073
+
1074
+ @overload
1075
+ def _get_cache_point_block(
1076
+ self,
1077
+ message_block: Union[
1078
+ OpenAIMessageContentListBlock,
1079
+ ChatCompletionUserMessage,
1080
+ ChatCompletionSystemMessage,
1081
+ ChatCompletionAssistantMessage,
1082
+ ],
1083
+ block_type: Literal["system"],
1084
+ model: Optional[str] = None,
1085
+ ) -> Optional[SystemContentBlock]:
1086
+ pass
1087
+
1088
+ @overload
1089
+ def _get_cache_point_block(
1090
+ self,
1091
+ message_block: Union[
1092
+ OpenAIMessageContentListBlock,
1093
+ ChatCompletionUserMessage,
1094
+ ChatCompletionSystemMessage,
1095
+ ChatCompletionAssistantMessage,
1096
+ ],
1097
+ block_type: Literal["content_block"],
1098
+ model: Optional[str] = None,
1099
+ ) -> Optional[ContentBlock]:
1100
+ pass
1101
+
1102
+ def _get_cache_point_block(
1103
+ self,
1104
+ message_block: Union[
1105
+ OpenAIMessageContentListBlock,
1106
+ ChatCompletionUserMessage,
1107
+ ChatCompletionSystemMessage,
1108
+ ChatCompletionAssistantMessage,
1109
+ ],
1110
+ block_type: Literal["system", "content_block"],
1111
+ model: Optional[str] = None,
1112
+ ) -> Optional[Union[SystemContentBlock, ContentBlock]]:
1113
+ cache_control = message_block.get("cache_control", None)
1114
+ if cache_control is None:
1115
+ return None
1116
+
1117
+ cache_point = CachePointBlock(type="default")
1118
+ if isinstance(cache_control, dict) and "ttl" in cache_control:
1119
+ ttl = cache_control["ttl"]
1120
+ if ttl in ["5m", "1h"] and model is not None:
1121
+ if is_claude_4_5_on_bedrock(model):
1122
+ cache_point["ttl"] = ttl
1123
+
1124
+ if block_type == "system":
1125
+ return SystemContentBlock(cachePoint=cache_point)
1126
+ else:
1127
+ return ContentBlock(cachePoint=cache_point)
1128
+
1129
+ def _transform_system_message(
1130
+ self, messages: List[AllMessageValues], model: Optional[str] = None
1131
+ ) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
1132
+ system_prompt_indices = []
1133
+ system_content_blocks: List[SystemContentBlock] = []
1134
+ for idx, message in enumerate(messages):
1135
+ if message["role"] == "system":
1136
+ system_prompt_indices.append(idx)
1137
+ if isinstance(message["content"], str) and message["content"]:
1138
+ system_content_blocks.append(
1139
+ SystemContentBlock(text=message["content"])
1140
+ )
1141
+ cache_block = self._get_cache_point_block(
1142
+ message, block_type="system", model=model
1143
+ )
1144
+ if cache_block:
1145
+ system_content_blocks.append(cache_block)
1146
+ elif isinstance(message["content"], list):
1147
+ for m in message["content"]:
1148
+ if m.get("type") == "text" and m.get("text"):
1149
+ system_content_blocks.append(
1150
+ SystemContentBlock(text=m["text"])
1151
+ )
1152
+ cache_block = self._get_cache_point_block(
1153
+ m, block_type="system", model=model
1154
+ )
1155
+ if cache_block:
1156
+ system_content_blocks.append(cache_block)
1157
+ if len(system_prompt_indices) > 0:
1158
+ for idx in reversed(system_prompt_indices):
1159
+ messages.pop(idx)
1160
+ return messages, system_content_blocks
1161
+
1162
+ def _transform_inference_params(self, inference_params: dict) -> InferenceConfig:
1163
+ if "top_k" in inference_params:
1164
+ inference_params["topK"] = inference_params.pop("top_k")
1165
+ return InferenceConfig(**inference_params)
1166
+
1167
+ def _handle_top_k_value(self, model: str, inference_params: dict) -> dict:
1168
+ base_model = BedrockModelInfo.get_base_model(model)
1169
+
1170
+ val_top_k = None
1171
+ if "topK" in inference_params:
1172
+ val_top_k = inference_params.pop("topK")
1173
+ elif "top_k" in inference_params:
1174
+ val_top_k = inference_params.pop("top_k")
1175
+
1176
+ if val_top_k:
1177
+ if base_model.startswith("anthropic"):
1178
+ return {"top_k": val_top_k}
1179
+ if base_model.startswith("amazon.nova"):
1180
+ return {"inferenceConfig": {"topK": val_top_k}}
1181
+
1182
+ return {}
1183
+
1184
+ def _prepare_request_params(
1185
+ self, optional_params: dict, model: str
1186
+ ) -> Tuple[dict, dict, dict, Optional[OutputConfigBlock]]:
1187
+ """Prepare and separate request parameters."""
1188
+ # Filter out exception objects before deepcopy to prevent deepcopy failures
1189
+ # Exceptions should not be stored in optional_params (this is a defensive fix)
1190
+ cleaned_params = filter_exceptions_from_params(optional_params)
1191
+ inference_params = safe_deep_copy(cleaned_params)
1192
+ supported_converse_params = list(
1193
+ AmazonConverseConfig.__annotations__.keys()
1194
+ ) + ["top_k"]
1195
+ supported_tool_call_params = ["tools", "tool_choice"]
1196
+ supported_config_params = list(self.get_config_blocks().keys())
1197
+ total_supported_params = (
1198
+ supported_converse_params
1199
+ + supported_tool_call_params
1200
+ + supported_config_params
1201
+ )
1202
+ inference_params.pop("json_mode", None) # used for handling json_schema
1203
+ # Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and
1204
+ # will reject `output_config` if it leaks through pass-through routes.
1205
+ inference_params.pop("output_config", None)
1206
+
1207
+ # Extract requestMetadata before processing other parameters
1208
+ request_metadata = inference_params.pop("requestMetadata", None)
1209
+ if request_metadata is not None:
1210
+ self._validate_request_metadata(request_metadata)
1211
+
1212
+ output_config: Optional[OutputConfigBlock] = inference_params.pop(
1213
+ "outputConfig", None
1214
+ )
1215
+ inference_params.pop(
1216
+ "output_config", None
1217
+ ) # Bedrock Converse doesn't support it
1218
+
1219
+ # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params'
1220
+ additional_request_params = {
1221
+ k: v for k, v in inference_params.items() if k not in total_supported_params
1222
+ }
1223
+ inference_params = {
1224
+ k: v for k, v in inference_params.items() if k in total_supported_params
1225
+ }
1226
+
1227
+ # Handle parallel_tool_calls configuration
1228
+ parallel_tool_use_config = additional_request_params.pop(
1229
+ "_parallel_tool_use_config", None
1230
+ )
1231
+ if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
1232
+ for key, value in parallel_tool_use_config.items():
1233
+ if (
1234
+ key in additional_request_params
1235
+ and isinstance(additional_request_params[key], dict)
1236
+ and isinstance(value, dict)
1237
+ ):
1238
+ additional_request_params[key].update(value)
1239
+ else:
1240
+ additional_request_params[key] = value
1241
+
1242
+ additional_request_params.pop("parallel_tool_calls", None)
1243
+
1244
+ # Only set the topK value in for models that support it
1245
+ additional_request_params.update(
1246
+ self._handle_top_k_value(model, inference_params)
1247
+ )
1248
+
1249
+ # Filter out internal/MCP-related parameters that shouldn't be sent to the API
1250
+ # These are LiteLLM internal parameters, not API parameters
1251
+ additional_request_params = filter_internal_params(additional_request_params)
1252
+
1253
+ # Filter out non-serializable objects (exceptions, callables, logging objects, etc.)
1254
+ # from additional_request_params to prevent JSON serialization errors
1255
+ # This filters: Exception objects, callable objects (functions), Logging objects, etc.
1256
+ additional_request_params = filter_exceptions_from_params(
1257
+ additional_request_params
1258
+ )
1259
+
1260
+ return (
1261
+ inference_params,
1262
+ additional_request_params,
1263
+ request_metadata,
1264
+ output_config,
1265
+ )
1266
+
1267
+ def _process_tools_and_beta(
1268
+ self,
1269
+ original_tools: list,
1270
+ model: str,
1271
+ headers: Optional[dict],
1272
+ additional_request_params: dict,
1273
+ ) -> Tuple[List[ToolBlock], list]:
1274
+ """Process tools and collect anthropic_beta values."""
1275
+ bedrock_tools: List[ToolBlock] = []
1276
+
1277
+ # Collect anthropic_beta values from user headers
1278
+ anthropic_beta_list = []
1279
+ if headers:
1280
+ user_betas = get_anthropic_beta_from_headers(headers)
1281
+ anthropic_beta_list.extend(user_betas)
1282
+
1283
+ # Separate pre-formatted Bedrock tools (e.g. systemTool from web_search_options)
1284
+ # from OpenAI-format tools that need transformation via _bedrock_tools_pt
1285
+ filtered_tools = []
1286
+ pre_formatted_tools: List[ToolBlock] = []
1287
+ if original_tools:
1288
+ for tool in original_tools:
1289
+ # Already-formatted Bedrock tools (e.g. systemTool for Nova grounding)
1290
+ if "systemTool" in tool:
1291
+ pre_formatted_tools.append(tool)
1292
+ continue
1293
+ tool_type = tool.get("type", "")
1294
+ if tool_type in (
1295
+ "tool_search_tool_regex_20251119",
1296
+ "tool_search_tool_bm25_20251119",
1297
+ ):
1298
+ # Tool search not supported in Converse API - skip it
1299
+ continue
1300
+ filtered_tools.append(tool)
1301
+
1302
+ # Only separate tools if computer use tools are actually present
1303
+ if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model):
1304
+ # Separate computer use tools from regular function tools
1305
+ computer_use_tools, regular_tools = self._separate_computer_use_tools(
1306
+ filtered_tools, model
1307
+ )
1308
+
1309
+ # Process regular function tools using existing logic
1310
+ bedrock_tools = _bedrock_tools_pt(regular_tools)
1311
+
1312
+ # Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
1313
+ if computer_use_tools:
1314
+ # Determine the correct computer-use beta header based on model
1315
+ # "computer-use-2025-11-24" for Claude Opus 4.6, Claude Opus 4.5
1316
+ # "computer-use-2025-01-24" for Claude Sonnet 4.5, Haiku 4.5, Opus 4.1, Sonnet 4, Opus 4, and Sonnet 3.7
1317
+ # "computer-use-2024-10-22" for older models
1318
+ model_lower = model.lower()
1319
+ if (
1320
+ "opus-4.6" in model_lower
1321
+ or "opus_4.6" in model_lower
1322
+ or "opus-4-6" in model_lower
1323
+ or "opus_4_6" in model_lower
1324
+ or "sonnet-4.6" in model_lower
1325
+ or "sonnet_4.6" in model_lower
1326
+ or "sonnet-4-6" in model_lower
1327
+ or "sonnet_4_6" in model_lower
1328
+ ):
1329
+ computer_use_header = "computer-use-2025-11-24"
1330
+ elif (
1331
+ "opus-4.5" in model_lower
1332
+ or "opus_4.5" in model_lower
1333
+ or "opus-4-5" in model_lower
1334
+ or "opus_4_5" in model_lower
1335
+ ):
1336
+ computer_use_header = "computer-use-2025-11-24"
1337
+ elif any(
1338
+ pattern in model_lower
1339
+ for pattern in [
1340
+ "sonnet-4.5",
1341
+ "sonnet_4.5",
1342
+ "sonnet-4-5",
1343
+ "sonnet_4_5",
1344
+ "haiku-4.5",
1345
+ "haiku_4.5",
1346
+ "haiku-4-5",
1347
+ "haiku_4_5",
1348
+ "opus-4.1",
1349
+ "opus_4.1",
1350
+ "opus-4-1",
1351
+ "opus_4_1",
1352
+ "sonnet-4",
1353
+ "sonnet_4",
1354
+ "opus-4",
1355
+ "opus_4",
1356
+ "sonnet-3.7",
1357
+ "sonnet_3.7",
1358
+ "sonnet-3-7",
1359
+ "sonnet_3_7",
1360
+ ]
1361
+ ):
1362
+ computer_use_header = "computer-use-2025-01-24"
1363
+ else:
1364
+ computer_use_header = "computer-use-2024-10-22"
1365
+
1366
+ anthropic_beta_list.append(computer_use_header)
1367
+ # Transform computer use tools to proper Bedrock format
1368
+ transformed_computer_tools = self._transform_computer_use_tools(
1369
+ computer_use_tools
1370
+ )
1371
+ additional_request_params["tools"] = transformed_computer_tools
1372
+ else:
1373
+ # No computer use tools, process all tools as regular tools
1374
+ bedrock_tools = _bedrock_tools_pt(filtered_tools)
1375
+
1376
+ # Append pre-formatted tools (systemTool etc.) after transformation
1377
+ bedrock_tools.extend(pre_formatted_tools)
1378
+
1379
+ # Set anthropic_beta in additional_request_params if we have any beta features
1380
+ # ONLY apply to Anthropic/Claude models - other models (e.g., Qwen, Llama) don't support this field
1381
+ base_model = BedrockModelInfo.get_base_model(model)
1382
+ if anthropic_beta_list and base_model.startswith("anthropic"):
1383
+ additional_request_params["anthropic_beta"] = anthropic_beta_list
1384
+
1385
+ return bedrock_tools, anthropic_beta_list
1386
+
1387
+ def _transform_request_helper(
1388
+ self,
1389
+ model: str,
1390
+ system_content_blocks: List[SystemContentBlock],
1391
+ optional_params: dict,
1392
+ messages: Optional[List[AllMessageValues]] = None,
1393
+ headers: Optional[dict] = None,
1394
+ ) -> CommonRequestObject:
1395
+ ## VALIDATE REQUEST
1396
+ """
1397
+ Bedrock doesn't support tool calling without `tools=` param specified.
1398
+ """
1399
+ if (
1400
+ "tools" not in optional_params
1401
+ and messages is not None
1402
+ and has_tool_call_blocks(messages)
1403
+ ):
1404
+ if litellm.modify_params:
1405
+ optional_params["tools"] = add_dummy_tool(
1406
+ custom_llm_provider="bedrock_converse"
1407
+ )
1408
+ else:
1409
+ raise litellm.UnsupportedParamsError(
1410
+ message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
1411
+ model="",
1412
+ llm_provider="bedrock",
1413
+ )
1414
+
1415
+ # Drop thinking param if thinking is enabled but thinking_blocks are missing
1416
+ # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
1417
+ #
1418
+ # IMPORTANT: Only drop thinking if NO assistant messages have thinking_blocks.
1419
+ # If any message has thinking_blocks, we must keep thinking enabled, otherwise
1420
+ # Related issues: https://github.com/BerriAI/litellm/issues/14194
1421
+ if (
1422
+ optional_params.get("thinking") is not None
1423
+ and messages is not None
1424
+ and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
1425
+ and not any_assistant_message_has_thinking_blocks(messages)
1426
+ ):
1427
+ if litellm.modify_params:
1428
+ optional_params.pop("thinking", None)
1429
+ litellm.verbose_logger.warning(
1430
+ "Dropping 'thinking' param because the last assistant message with tool_calls "
1431
+ "has no thinking_blocks. The model won't use extended thinking for this turn."
1432
+ )
1433
+
1434
+ # Prepare and separate parameters
1435
+ (
1436
+ inference_params,
1437
+ additional_request_params,
1438
+ request_metadata,
1439
+ output_config,
1440
+ ) = self._prepare_request_params(optional_params, model)
1441
+
1442
+ original_tools = inference_params.pop("tools", [])
1443
+
1444
+ # Process tools and collect beta values
1445
+ bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
1446
+ original_tools, model, headers, additional_request_params
1447
+ )
1448
+
1449
+ # Append cachePoint to tools if cache_control_injection_points has tool_config
1450
+ cache_injection_points = additional_request_params.pop(
1451
+ "cache_control_injection_points", None
1452
+ )
1453
+ if cache_injection_points and len(bedrock_tools) > 0:
1454
+ for point in cache_injection_points:
1455
+ if point.get("location") == "tool_config":
1456
+ bedrock_tools.append({"cachePoint": {"type": "default"}})
1457
+ break
1458
+
1459
+ bedrock_tool_config: Optional[ToolConfigBlock] = None
1460
+ if len(bedrock_tools) > 0:
1461
+ tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
1462
+ "tool_choice", None
1463
+ )
1464
+ bedrock_tool_config = ToolConfigBlock(
1465
+ tools=bedrock_tools,
1466
+ )
1467
+ if tool_choice_values is not None:
1468
+ bedrock_tool_config["toolChoice"] = tool_choice_values
1469
+
1470
+ data: CommonRequestObject = {
1471
+ "additionalModelRequestFields": additional_request_params,
1472
+ "system": system_content_blocks,
1473
+ "inferenceConfig": self._transform_inference_params(
1474
+ inference_params=inference_params
1475
+ ),
1476
+ }
1477
+
1478
+ # Handle all config blocks
1479
+ for config_name, config_class in self.get_config_blocks().items():
1480
+ config_value = inference_params.pop(config_name, None)
1481
+ if config_value is not None:
1482
+ data[config_name] = config_class(**config_value) # type: ignore
1483
+
1484
+ # Tool Config
1485
+ if bedrock_tool_config is not None:
1486
+ data["toolConfig"] = bedrock_tool_config
1487
+
1488
+ # Request Metadata (top-level field)
1489
+ if request_metadata is not None:
1490
+ data["requestMetadata"] = request_metadata
1491
+
1492
+ if output_config is not None:
1493
+ data["outputConfig"] = output_config
1494
+
1495
+ return data
1496
+
1497
+ async def _async_transform_request(
1498
+ self,
1499
+ model: str,
1500
+ messages: List[AllMessageValues],
1501
+ optional_params: dict,
1502
+ litellm_params: dict,
1503
+ headers: Optional[dict] = None,
1504
+ ) -> RequestObject:
1505
+ messages, system_content_blocks = self._transform_system_message(
1506
+ messages, model=model
1507
+ )
1508
+
1509
+ # Convert last user message to guarded_text if guardrailConfig is present
1510
+ messages = self._convert_consecutive_user_messages_to_guarded_text(
1511
+ messages, optional_params
1512
+ )
1513
+ ## TRANSFORMATION ##
1514
+
1515
+ _data: CommonRequestObject = self._transform_request_helper(
1516
+ model=model,
1517
+ system_content_blocks=system_content_blocks,
1518
+ optional_params=optional_params,
1519
+ messages=messages,
1520
+ headers=headers,
1521
+ )
1522
+
1523
+ bedrock_messages = (
1524
+ await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
1525
+ messages=messages,
1526
+ model=model,
1527
+ llm_provider="bedrock_converse",
1528
+ user_continue_message=litellm_params.pop("user_continue_message", None),
1529
+ )
1530
+ )
1531
+
1532
+ data: RequestObject = {"messages": bedrock_messages, **_data}
1533
+
1534
+ return data
1535
+
1536
+ def transform_request(
1537
+ self,
1538
+ model: str,
1539
+ messages: List[AllMessageValues],
1540
+ optional_params: dict,
1541
+ litellm_params: dict,
1542
+ headers: dict,
1543
+ ) -> dict:
1544
+ return cast(
1545
+ dict,
1546
+ self._transform_request(
1547
+ model=model,
1548
+ messages=messages,
1549
+ optional_params=optional_params,
1550
+ litellm_params=litellm_params,
1551
+ headers=headers,
1552
+ ),
1553
+ )
1554
+
1555
+ def _transform_request(
1556
+ self,
1557
+ model: str,
1558
+ messages: List[AllMessageValues],
1559
+ optional_params: dict,
1560
+ litellm_params: dict,
1561
+ headers: Optional[dict] = None,
1562
+ ) -> RequestObject:
1563
+ messages, system_content_blocks = self._transform_system_message(
1564
+ messages, model=model
1565
+ )
1566
+
1567
+ # Convert last user message to guarded_text if guardrailConfig is present
1568
+ messages = self._convert_consecutive_user_messages_to_guarded_text(
1569
+ messages, optional_params
1570
+ )
1571
+
1572
+ _data: CommonRequestObject = self._transform_request_helper(
1573
+ model=model,
1574
+ system_content_blocks=system_content_blocks,
1575
+ optional_params=optional_params,
1576
+ messages=messages,
1577
+ headers=headers,
1578
+ )
1579
+
1580
+ ## TRANSFORMATION ##
1581
+ bedrock_messages: List[MessageBlock] = _bedrock_converse_messages_pt(
1582
+ messages=messages,
1583
+ model=model,
1584
+ llm_provider="bedrock_converse",
1585
+ user_continue_message=litellm_params.pop("user_continue_message", None),
1586
+ )
1587
+
1588
+ data: RequestObject = {"messages": bedrock_messages, **_data}
1589
+
1590
+ return data
1591
+
1592
+ def transform_response(
1593
+ self,
1594
+ model: str,
1595
+ raw_response: httpx.Response,
1596
+ model_response: ModelResponse,
1597
+ logging_obj: Logging,
1598
+ request_data: dict,
1599
+ messages: List[AllMessageValues],
1600
+ optional_params: dict,
1601
+ litellm_params: dict,
1602
+ encoding: Any,
1603
+ api_key: Optional[str] = None,
1604
+ json_mode: Optional[bool] = None,
1605
+ ) -> ModelResponse:
1606
+ return self._transform_response(
1607
+ model=model,
1608
+ response=raw_response,
1609
+ model_response=model_response,
1610
+ stream=optional_params.get("stream", False),
1611
+ logging_obj=logging_obj,
1612
+ optional_params=optional_params,
1613
+ api_key=api_key,
1614
+ data=request_data,
1615
+ messages=messages,
1616
+ encoding=encoding,
1617
+ )
1618
+
1619
+ def _transform_reasoning_content(
1620
+ self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]
1621
+ ) -> str:
1622
+ """
1623
+ Extract the reasoning text from the reasoning content blocks
1624
+
1625
+ Ensures deepseek reasoning content compatible output.
1626
+ """
1627
+ reasoning_content_str = ""
1628
+ for block in reasoning_content_blocks:
1629
+ if "reasoningText" in block:
1630
+ reasoning_content_str += block["reasoningText"]["text"]
1631
+ return reasoning_content_str
1632
+
1633
+ def _transform_thinking_blocks(
1634
+ self, thinking_blocks: List[BedrockConverseReasoningContentBlock]
1635
+ ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]:
1636
+ """Return a consistent format for thinking blocks between Anthropic and Bedrock."""
1637
+ thinking_blocks_list: List[
1638
+ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
1639
+ ] = []
1640
+ for block in thinking_blocks:
1641
+ if "reasoningText" in block:
1642
+ _thinking_block = ChatCompletionThinkingBlock(type="thinking")
1643
+ _text = block["reasoningText"].get("text")
1644
+ _signature = block["reasoningText"].get("signature")
1645
+ if _text is not None:
1646
+ _thinking_block["thinking"] = _text
1647
+ if _signature is not None:
1648
+ _thinking_block["signature"] = _signature
1649
+ thinking_blocks_list.append(_thinking_block)
1650
+ elif "redactedContent" in block:
1651
+ _redacted_block = ChatCompletionRedactedThinkingBlock(
1652
+ type="redacted_thinking", data=block["redactedContent"]
1653
+ )
1654
+ thinking_blocks_list.append(_redacted_block)
1655
+ return thinking_blocks_list
1656
+
1657
+ def _transform_usage(
1658
+ self,
1659
+ usage: ConverseTokenUsageBlock,
1660
+ reasoning_content: Optional[str] = None,
1661
+ ) -> Usage:
1662
+ input_tokens = usage["inputTokens"]
1663
+ output_tokens = usage["outputTokens"]
1664
+ total_tokens = usage["totalTokens"]
1665
+ cache_creation_input_tokens: int = 0
1666
+ cache_read_input_tokens: int = 0
1667
+
1668
+ if "cacheReadInputTokens" in usage:
1669
+ cache_read_input_tokens = usage["cacheReadInputTokens"]
1670
+ input_tokens += cache_read_input_tokens
1671
+ if "cacheWriteInputTokens" in usage:
1672
+ cache_creation_input_tokens = usage["cacheWriteInputTokens"]
1673
+ input_tokens += cache_creation_input_tokens
1674
+
1675
+ prompt_tokens_details = PromptTokensDetailsWrapper(
1676
+ cached_tokens=cache_read_input_tokens
1677
+ )
1678
+ reasoning_tokens = (
1679
+ token_counter(text=reasoning_content, count_response_tokens=True)
1680
+ if reasoning_content
1681
+ else 0
1682
+ )
1683
+ completion_tokens_details = CompletionTokensDetailsWrapper(
1684
+ reasoning_tokens=reasoning_tokens,
1685
+ text_tokens=(
1686
+ output_tokens - reasoning_tokens
1687
+ if reasoning_tokens > 0
1688
+ else output_tokens
1689
+ ),
1690
+ )
1691
+ openai_usage = Usage(
1692
+ prompt_tokens=input_tokens,
1693
+ completion_tokens=output_tokens,
1694
+ total_tokens=total_tokens,
1695
+ prompt_tokens_details=prompt_tokens_details,
1696
+ cache_creation_input_tokens=cache_creation_input_tokens,
1697
+ cache_read_input_tokens=cache_read_input_tokens,
1698
+ completion_tokens_details=completion_tokens_details,
1699
+ )
1700
+ return openai_usage
1701
+
1702
+ def get_tool_call_names(
1703
+ self,
1704
+ tools: Optional[
1705
+ Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]
1706
+ ] = None,
1707
+ ) -> List[str]:
1708
+ if tools is None:
1709
+ return []
1710
+ tool_set: set[str] = set()
1711
+ for tool in tools:
1712
+ tool_spec = tool.get("toolSpec")
1713
+ function = tool.get("function")
1714
+ if tool_spec is not None:
1715
+ _name = cast(dict, tool_spec).get("name")
1716
+ if _name is not None and isinstance(_name, str):
1717
+ tool_set.add(_name)
1718
+ if function is not None:
1719
+ _name = cast(dict, function).get("name")
1720
+ if _name is not None and isinstance(_name, str):
1721
+ tool_set.add(_name)
1722
+ return list(tool_set)
1723
+
1724
+ def apply_tool_call_transformation_if_needed(
1725
+ self,
1726
+ message: Message,
1727
+ tools: Optional[List[ToolBlock]] = None,
1728
+ initial_finish_reason: Optional[str] = None,
1729
+ ) -> Tuple[Message, Optional[str]]:
1730
+ """
1731
+ Apply tool call transformation to a message.
1732
+
1733
+ LLM providers (e.g. Bedrock, Vertex AI) sometimes return tool call in the response content.
1734
+
1735
+ If the response content is a JSON object, we can parse it and return the tool call in the tool_calls field.
1736
+ """
1737
+ returned_finish_reason = initial_finish_reason
1738
+ if tools is None:
1739
+ return message, returned_finish_reason
1740
+
1741
+ if message.content is not None:
1742
+ try:
1743
+ tool_call_names = self.get_tool_call_names(tools)
1744
+ json_content = json.loads(message.content)
1745
+ if (
1746
+ json_content.get("type") == "function"
1747
+ and json_content.get("name") in tool_call_names
1748
+ ):
1749
+ tool_calls = [
1750
+ ChatCompletionMessageToolCall(function=Function(**json_content))
1751
+ ]
1752
+
1753
+ message.tool_calls = tool_calls
1754
+ message.content = None
1755
+ returned_finish_reason = "tool_calls"
1756
+ except Exception:
1757
+ pass
1758
+
1759
+ return message, returned_finish_reason
1760
+
1761
+ def _translate_message_content(
1762
+ self, content_blocks: List[ContentBlock]
1763
+ ) -> Tuple[
1764
+ str,
1765
+ List[ChatCompletionToolCallChunk],
1766
+ Optional[List[BedrockConverseReasoningContentBlock]],
1767
+ Optional[List[CitationsContentBlock]],
1768
+ ]:
1769
+ """
1770
+ Translate the message content to a string and a list of tool calls, reasoning content blocks, and citations.
1771
+
1772
+ Returns:
1773
+ content_str: str
1774
+ tools: List[ChatCompletionToolCallChunk]
1775
+ reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]]
1776
+ citationsContentBlocks: Optional[List[CitationsContentBlock]] - Citations from Nova grounding
1777
+ """
1778
+ content_str = ""
1779
+ tools: List[ChatCompletionToolCallChunk] = []
1780
+ reasoningContentBlocks: Optional[
1781
+ List[BedrockConverseReasoningContentBlock]
1782
+ ] = None
1783
+ citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
1784
+ for idx, content in enumerate(content_blocks):
1785
+ """
1786
+ - Content is either a tool response or text
1787
+ """
1788
+ extracted_reasoning_content_str: Optional[str] = None
1789
+ if "text" in content:
1790
+ (
1791
+ extracted_reasoning_content_str,
1792
+ _content_str,
1793
+ ) = _parse_content_for_reasoning(content["text"])
1794
+ if _content_str is not None:
1795
+ content_str += _content_str
1796
+ if "toolUse" in content:
1797
+ ## check tool name was formatted by litellm
1798
+ _response_tool_name = content["toolUse"]["name"]
1799
+ response_tool_name = get_bedrock_tool_name(
1800
+ response_tool_name=_response_tool_name
1801
+ )
1802
+ _function_chunk = ChatCompletionToolCallFunctionChunk(
1803
+ name=response_tool_name,
1804
+ arguments=json.dumps(content["toolUse"]["input"]),
1805
+ )
1806
+
1807
+ _tool_response_chunk = ChatCompletionToolCallChunk(
1808
+ id=content["toolUse"]["toolUseId"],
1809
+ type="function",
1810
+ function=_function_chunk,
1811
+ index=idx,
1812
+ )
1813
+ tools.append(_tool_response_chunk)
1814
+ if extracted_reasoning_content_str is not None:
1815
+ if reasoningContentBlocks is None:
1816
+ reasoningContentBlocks = []
1817
+ reasoningContentBlocks.append(
1818
+ BedrockConverseReasoningContentBlock(
1819
+ reasoningText=BedrockConverseReasoningTextBlock(
1820
+ text=extracted_reasoning_content_str,
1821
+ )
1822
+ )
1823
+ )
1824
+ if "reasoningContent" in content:
1825
+ if reasoningContentBlocks is None:
1826
+ reasoningContentBlocks = []
1827
+ reasoningContentBlocks.append(content["reasoningContent"])
1828
+ # Handle Nova grounding citations content
1829
+ if "citationsContent" in content:
1830
+ if citationsContentBlocks is None:
1831
+ citationsContentBlocks = []
1832
+ citationsContentBlocks.append(content["citationsContent"])
1833
+
1834
+ return content_str, tools, reasoningContentBlocks, citationsContentBlocks
1835
+
1836
+ @staticmethod
1837
+ def _unwrap_bedrock_properties(json_str: str) -> str:
1838
+ """
1839
+ Unwrap Bedrock's response_format JSON structure.
1840
+
1841
+ If the JSON has a single "properties" key, extract its value.
1842
+ Otherwise, return the original string.
1843
+
1844
+ Args:
1845
+ json_str: JSON string to unwrap
1846
+
1847
+ Returns:
1848
+ Unwrapped JSON string or original if unwrapping not needed
1849
+ """
1850
+ try:
1851
+ response_data = json.loads(json_str)
1852
+ if (
1853
+ isinstance(response_data, dict)
1854
+ and "properties" in response_data
1855
+ and len(response_data) == 1
1856
+ ):
1857
+ response_data = response_data["properties"]
1858
+ return json.dumps(response_data)
1859
+ except json.JSONDecodeError:
1860
+ pass
1861
+ return json_str
1862
+
1863
+ @staticmethod
1864
+ def _filter_json_mode_tools(
1865
+ json_mode: Optional[bool],
1866
+ tools: List[ChatCompletionToolCallChunk],
1867
+ chat_completion_message: ChatCompletionResponseMessage,
1868
+ ) -> Optional[List[ChatCompletionToolCallChunk]]:
1869
+ """
1870
+ When json_mode is True, Bedrock may return the internal `json_tool_call`
1871
+ tool alongside real user-defined tools. This method handles 3 scenarios:
1872
+
1873
+ 1. Only json_tool_call present -> convert to text content, return None
1874
+ 2. Mixed json_tool_call + real -> filter out json_tool_call, return real tools
1875
+ 3. No json_tool_call / no json_mode -> return tools as-is
1876
+ """
1877
+ if not json_mode or not tools:
1878
+ return tools if tools else None
1879
+
1880
+ json_tool_indices = [
1881
+ i
1882
+ for i, t in enumerate(tools)
1883
+ if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME
1884
+ ]
1885
+
1886
+ if not json_tool_indices:
1887
+ # No json_tool_call found, return tools unchanged
1888
+ return tools
1889
+
1890
+ if len(json_tool_indices) == len(tools):
1891
+ # All tools are json_tool_call — convert first one to content
1892
+ verbose_logger.debug(
1893
+ "Processing JSON tool call response for response_format"
1894
+ )
1895
+ json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments")
1896
+ if json_mode_content_str is not None:
1897
+ json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(
1898
+ json_mode_content_str
1899
+ )
1900
+ chat_completion_message["content"] = json_mode_content_str
1901
+ return None
1902
+
1903
+ # Mixed: filter out json_tool_call, keep real tools.
1904
+ # Preserve the json_tool_call content as message text so the structured
1905
+ # output from response_format is not silently lost.
1906
+ first_idx = json_tool_indices[0]
1907
+ json_mode_args = tools[first_idx]["function"].get("arguments")
1908
+ if json_mode_args is not None:
1909
+ json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(
1910
+ json_mode_args
1911
+ )
1912
+ existing = chat_completion_message.get("content") or ""
1913
+ chat_completion_message["content"] = (
1914
+ existing + json_mode_args if existing else json_mode_args
1915
+ )
1916
+
1917
+ real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices]
1918
+ return real_tools if real_tools else None
1919
+
1920
+ def _transform_response( # noqa: PLR0915
1921
+ self,
1922
+ model: str,
1923
+ response: httpx.Response,
1924
+ model_response: ModelResponse,
1925
+ stream: bool,
1926
+ logging_obj: Optional[Logging],
1927
+ optional_params: dict,
1928
+ api_key: Optional[str],
1929
+ data: Union[dict, str],
1930
+ messages: List,
1931
+ encoding,
1932
+ ) -> ModelResponse:
1933
+ ## LOGGING
1934
+ if logging_obj is not None:
1935
+ logging_obj.post_call(
1936
+ input=messages,
1937
+ api_key=api_key,
1938
+ original_response=response.text,
1939
+ additional_args={"complete_input_dict": data},
1940
+ )
1941
+
1942
+ json_mode: Optional[bool] = optional_params.get("json_mode", None)
1943
+ ## RESPONSE OBJECT
1944
+ try:
1945
+ completion_response = ConverseResponseBlock(**response.json()) # type: ignore
1946
+ except Exception as e:
1947
+ raise BedrockError(
1948
+ message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
1949
+ response.text, str(e)
1950
+ ),
1951
+ status_code=422,
1952
+ )
1953
+
1954
+ """
1955
+ Bedrock Response Object has optional message block
1956
+
1957
+ completion_response["output"].get("message", None)
1958
+
1959
+ A message block looks like this (Example 1):
1960
+ "output": {
1961
+ "message": {
1962
+ "role": "assistant",
1963
+ "content": [
1964
+ {
1965
+ "text": "Is there anything else you'd like to talk about? Perhaps I can help with some economic questions or provide some information about economic concepts?"
1966
+ }
1967
+ ]
1968
+ }
1969
+ },
1970
+ (Example 2):
1971
+ "output": {
1972
+ "message": {
1973
+ "role": "assistant",
1974
+ "content": [
1975
+ {
1976
+ "toolUse": {
1977
+ "toolUseId": "tooluse_hbTgdi0CSLq_hM4P8csZJA",
1978
+ "name": "top_song",
1979
+ "input": {
1980
+ "sign": "WZPZ"
1981
+ }
1982
+ }
1983
+ }
1984
+ ]
1985
+ }
1986
+ }
1987
+
1988
+ """
1989
+ message: Optional[MessageBlock] = completion_response["output"]["message"]
1990
+ chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
1991
+ content_str = ""
1992
+ tools: List[ChatCompletionToolCallChunk] = []
1993
+ reasoningContentBlocks: Optional[
1994
+ List[BedrockConverseReasoningContentBlock]
1995
+ ] = None
1996
+ citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
1997
+
1998
+ if message is not None:
1999
+ (
2000
+ content_str,
2001
+ tools,
2002
+ reasoningContentBlocks,
2003
+ citationsContentBlocks,
2004
+ ) = self._translate_message_content(message["content"])
2005
+
2006
+ # Initialize provider_specific_fields if we have any special content blocks
2007
+ provider_specific_fields: dict = {}
2008
+ if reasoningContentBlocks is not None:
2009
+ provider_specific_fields["reasoningContentBlocks"] = reasoningContentBlocks
2010
+ if citationsContentBlocks is not None:
2011
+ provider_specific_fields["citationsContent"] = citationsContentBlocks
2012
+
2013
+ if provider_specific_fields:
2014
+ chat_completion_message[
2015
+ "provider_specific_fields"
2016
+ ] = provider_specific_fields
2017
+
2018
+ if reasoningContentBlocks is not None:
2019
+ chat_completion_message[
2020
+ "reasoning_content"
2021
+ ] = self._transform_reasoning_content(reasoningContentBlocks)
2022
+ chat_completion_message[
2023
+ "thinking_blocks"
2024
+ ] = self._transform_thinking_blocks(reasoningContentBlocks)
2025
+ chat_completion_message["content"] = content_str
2026
+ filtered_tools = self._filter_json_mode_tools(
2027
+ json_mode=json_mode,
2028
+ tools=tools,
2029
+ chat_completion_message=chat_completion_message,
2030
+ )
2031
+ if filtered_tools:
2032
+ chat_completion_message["tool_calls"] = filtered_tools
2033
+
2034
+ ## CALCULATING USAGE - bedrock returns usage in the headers
2035
+ usage = self._transform_usage(
2036
+ completion_response["usage"],
2037
+ reasoning_content=chat_completion_message.get("reasoning_content"),
2038
+ )
2039
+
2040
+ ## HANDLE TOOL CALLS
2041
+ _message = Message(**chat_completion_message)
2042
+ initial_finish_reason = map_finish_reason(completion_response["stopReason"])
2043
+
2044
+ (
2045
+ returned_message,
2046
+ returned_finish_reason,
2047
+ ) = self.apply_tool_call_transformation_if_needed(
2048
+ message=_message,
2049
+ tools=optional_params.get("tools"),
2050
+ initial_finish_reason=initial_finish_reason,
2051
+ )
2052
+ model_response.choices = [
2053
+ litellm.Choices(
2054
+ finish_reason=returned_finish_reason,
2055
+ index=0,
2056
+ message=returned_message,
2057
+ )
2058
+ ]
2059
+ model_response.created = int(time.time())
2060
+ model_response.model = model
2061
+
2062
+ setattr(model_response, "usage", usage)
2063
+
2064
+ # Add "trace" from Bedrock guardrails - if user has opted in to returning it
2065
+ if "trace" in completion_response:
2066
+ setattr(model_response, "trace", completion_response["trace"])
2067
+
2068
+ # Add service_tier if present in Bedrock response
2069
+ # Map Bedrock serviceTier (object) to OpenAI service_tier (string)
2070
+ if "serviceTier" in completion_response:
2071
+ service_tier_block = completion_response["serviceTier"]
2072
+ if isinstance(service_tier_block, dict) and "type" in service_tier_block:
2073
+ setattr(model_response, "service_tier", service_tier_block["type"])
2074
+
2075
+ return model_response
2076
+
2077
+ def get_error_class(
2078
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
2079
+ ) -> BaseLLMException:
2080
+ return BedrockError(
2081
+ message=error_message,
2082
+ status_code=status_code,
2083
+ headers=headers,
2084
+ )
2085
+
2086
+ def validate_environment(
2087
+ self,
2088
+ headers: dict,
2089
+ model: str,
2090
+ messages: List[AllMessageValues],
2091
+ optional_params: dict,
2092
+ litellm_params: dict,
2093
+ api_key: Optional[str] = None,
2094
+ api_base: Optional[str] = None,
2095
+ ) -> dict:
2096
+ if api_key:
2097
+ headers["Authorization"] = f"Bearer {api_key}"
2098
+ return headers
2099
+
2100
+ def should_fake_stream(
2101
+ self,
2102
+ model: Optional[str],
2103
+ stream: Optional[bool],
2104
+ custom_llm_provider: Optional[str] = None,
2105
+ fake_stream: Optional[bool] = None,
2106
+ ) -> bool:
2107
+ """
2108
+ Returns True if the model/provider should fake stream
2109
+ """
2110
+ ###################################################################
2111
+ # If an upstream method already set fake_stream to True, return True
2112
+ ###################################################################
2113
+ if fake_stream is True:
2114
+ return True
2115
+
2116
+ ###################################################################
2117
+ # Bedrock Converse Specific Logic
2118
+ ###################################################################
2119
+ if stream is True:
2120
+ if model is not None:
2121
+ ###################################################################
2122
+ # AI21 models do not support streaming
2123
+ ###################################################################
2124
+ if "ai21" in model:
2125
+ return True
2126
+ return False
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_agent/transformation.py ADDED
@@ -0,0 +1,547 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation for Bedrock Invoke Agent
3
+
4
+ https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html
5
+ """
6
+
7
+ import base64
8
+ import json
9
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
10
+
11
+ import httpx
12
+
13
+ from litellm._logging import verbose_logger
14
+ from litellm._uuid import uuid
15
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
16
+ convert_content_list_to_str,
17
+ )
18
+ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
19
+ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
20
+ from litellm.llms.bedrock.common_utils import BedrockError
21
+ from litellm.types.llms.bedrock_invoke_agents import (
22
+ InvokeAgentChunkPayload,
23
+ InvokeAgentEvent,
24
+ InvokeAgentEventHeaders,
25
+ InvokeAgentEventList,
26
+ InvokeAgentMetadata,
27
+ InvokeAgentModelInvocationInput,
28
+ InvokeAgentModelInvocationOutput,
29
+ InvokeAgentOrchestrationTrace,
30
+ InvokeAgentPreProcessingTrace,
31
+ InvokeAgentTrace,
32
+ InvokeAgentTracePayload,
33
+ InvokeAgentUsage,
34
+ )
35
+ from litellm.types.llms.openai import AllMessageValues
36
+ from litellm.types.utils import Choices, Message, ModelResponse
37
+
38
+ if TYPE_CHECKING:
39
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
40
+
41
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
42
+ else:
43
+ LiteLLMLoggingObj = Any
44
+
45
+
46
+ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
47
+ def __init__(self, **kwargs):
48
+ BaseConfig.__init__(self, **kwargs)
49
+ BaseAWSLLM.__init__(self, **kwargs)
50
+
51
+ def get_supported_openai_params(self, model: str) -> List[str]:
52
+ """
53
+ This is a base invoke agent model mapping. For Invoke Agent - define a bedrock provider specific config that extends this class.
54
+
55
+ Bedrock Invoke Agents has 0 OpenAI compatible params
56
+
57
+ As of May 29th, 2025 - they don't support streaming.
58
+ """
59
+ return []
60
+
61
+ def map_openai_params(
62
+ self,
63
+ non_default_params: dict,
64
+ optional_params: dict,
65
+ model: str,
66
+ drop_params: bool,
67
+ ) -> dict:
68
+ """
69
+ This is a base invoke agent model mapping. For Invoke Agent - define a bedrock provider specific config that extends this class.
70
+ """
71
+ return optional_params
72
+
73
+ def get_complete_url(
74
+ self,
75
+ api_base: Optional[str],
76
+ api_key: Optional[str],
77
+ model: str,
78
+ optional_params: dict,
79
+ litellm_params: dict,
80
+ stream: Optional[bool] = None,
81
+ ) -> str:
82
+ """
83
+ Get the complete url for the request
84
+ """
85
+ ### SET RUNTIME ENDPOINT ###
86
+ aws_bedrock_runtime_endpoint = optional_params.get(
87
+ "aws_bedrock_runtime_endpoint", None
88
+ ) # https://bedrock-runtime.{region_name}.amazonaws.com
89
+ endpoint_url, _ = self.get_runtime_endpoint(
90
+ api_base=api_base,
91
+ aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
92
+ aws_region_name=self._get_aws_region_name(
93
+ optional_params=optional_params, model=model
94
+ ),
95
+ endpoint_type="agent",
96
+ )
97
+
98
+ agent_id, agent_alias_id = self._get_agent_id_and_alias_id(model)
99
+ session_id = self._get_session_id(optional_params)
100
+
101
+ endpoint_url = f"{endpoint_url}/agents/{agent_id}/agentAliases/{agent_alias_id}/sessions/{session_id}/text"
102
+
103
+ return endpoint_url
104
+
105
+ def sign_request(
106
+ self,
107
+ headers: dict,
108
+ optional_params: dict,
109
+ request_data: dict,
110
+ api_base: str,
111
+ api_key: Optional[str] = None,
112
+ model: Optional[str] = None,
113
+ stream: Optional[bool] = None,
114
+ fake_stream: Optional[bool] = None,
115
+ ) -> Tuple[dict, Optional[bytes]]:
116
+ return self._sign_request(
117
+ service_name="bedrock",
118
+ headers=headers,
119
+ optional_params=optional_params,
120
+ request_data=request_data,
121
+ api_base=api_base,
122
+ model=model,
123
+ stream=stream,
124
+ fake_stream=fake_stream,
125
+ api_key=api_key,
126
+ )
127
+
128
+ def _get_agent_id_and_alias_id(self, model: str) -> tuple[str, str]:
129
+ """
130
+ model = "agent/L1RT58GYRW/MFPSBCXYTW"
131
+ agent_id = "L1RT58GYRW"
132
+ agent_alias_id = "MFPSBCXYTW"
133
+ """
134
+ # Split the model string by '/' and extract components
135
+ parts = model.split("/")
136
+ if len(parts) != 3 or parts[0] != "agent":
137
+ raise ValueError(
138
+ "Invalid model format. Expected format: 'model=agent/AGENT_ID/ALIAS_ID'"
139
+ )
140
+
141
+ return parts[1], parts[2] # Return (agent_id, agent_alias_id)
142
+
143
+ def _get_session_id(self, optional_params: dict) -> str:
144
+ """ """
145
+ return optional_params.get("sessionID", None) or str(uuid.uuid4())
146
+
147
+ def transform_request(
148
+ self,
149
+ model: str,
150
+ messages: List[AllMessageValues],
151
+ optional_params: dict,
152
+ litellm_params: dict,
153
+ headers: dict,
154
+ ) -> dict:
155
+ # use the last message content as the query
156
+ query: str = convert_content_list_to_str(messages[-1])
157
+ return {
158
+ "inputText": query,
159
+ "enableTrace": True,
160
+ **optional_params,
161
+ }
162
+
163
+ def _parse_aws_event_stream(self, raw_content: bytes) -> InvokeAgentEventList:
164
+ """
165
+ Parse AWS event stream format using boto3/botocore's built-in parser.
166
+ This is the same approach used in the existing AWSEventStreamDecoder.
167
+ """
168
+ try:
169
+ from botocore.eventstream import EventStreamBuffer
170
+ from botocore.parsers import EventStreamJSONParser
171
+ except ImportError:
172
+ raise ImportError("boto3/botocore is required for AWS event stream parsing")
173
+
174
+ events: InvokeAgentEventList = []
175
+ parser = EventStreamJSONParser()
176
+ event_stream_buffer = EventStreamBuffer()
177
+
178
+ # Add the entire response to the buffer
179
+ event_stream_buffer.add_data(raw_content)
180
+
181
+ # Process all events in the buffer
182
+ for event in event_stream_buffer:
183
+ try:
184
+ headers = self._extract_headers_from_event(event)
185
+
186
+ event_type = headers.get("event_type", "")
187
+
188
+ if event_type == "chunk":
189
+ # Handle chunk events specially - they contain decoded content, not JSON
190
+ message = self._parse_message_from_event(event, parser)
191
+ parsed_event: InvokeAgentEvent = InvokeAgentEvent()
192
+ if message:
193
+ # For chunk events, create a payload with the decoded content
194
+ parsed_event = {
195
+ "headers": headers,
196
+ "payload": {
197
+ "bytes": base64.b64encode(
198
+ message.encode("utf-8")
199
+ ).decode("utf-8")
200
+ }, # Re-encode for consistency
201
+ }
202
+ events.append(parsed_event)
203
+
204
+ elif event_type == "trace":
205
+ # Handle trace events normally - they contain JSON
206
+ message = self._parse_message_from_event(event, parser)
207
+
208
+ if message:
209
+ try:
210
+ event_data = json.loads(message)
211
+ parsed_event = {
212
+ "headers": headers,
213
+ "payload": event_data,
214
+ }
215
+ events.append(parsed_event)
216
+ except json.JSONDecodeError as e:
217
+ verbose_logger.warning(
218
+ f"Failed to parse trace event JSON: {e}"
219
+ )
220
+ else:
221
+ verbose_logger.debug(f"Unknown event type: {event_type}")
222
+
223
+ except Exception as e:
224
+ verbose_logger.error(f"Error processing event: {e}")
225
+ continue
226
+
227
+ return events
228
+
229
+ def _parse_message_from_event(self, event, parser) -> Optional[str]:
230
+ """Extract message content from an AWS event, adapted from AWSEventStreamDecoder."""
231
+ try:
232
+ response_dict = event.to_response_dict()
233
+ verbose_logger.debug(f"Response dict: {response_dict}")
234
+
235
+ # Use the same response shape parsing as the existing decoder
236
+ parsed_response = parser.parse(
237
+ response_dict, self._get_response_stream_shape()
238
+ )
239
+ verbose_logger.debug(f"Parsed response: {parsed_response}")
240
+
241
+ if response_dict["status_code"] != 200:
242
+ decoded_body = response_dict["body"].decode()
243
+ if isinstance(decoded_body, dict):
244
+ error_message = decoded_body.get("message")
245
+ elif isinstance(decoded_body, str):
246
+ error_message = decoded_body
247
+ else:
248
+ error_message = ""
249
+ exception_status = response_dict["headers"].get(":exception-type")
250
+ error_message = exception_status + " " + error_message
251
+ raise BedrockError(
252
+ status_code=response_dict["status_code"],
253
+ message=(
254
+ json.dumps(error_message)
255
+ if isinstance(error_message, dict)
256
+ else error_message
257
+ ),
258
+ )
259
+
260
+ if "chunk" in parsed_response:
261
+ chunk = parsed_response.get("chunk")
262
+ if not chunk:
263
+ return None
264
+ return chunk.get("bytes").decode()
265
+ else:
266
+ chunk = response_dict.get("body")
267
+ if not chunk:
268
+ return None
269
+ return chunk.decode()
270
+
271
+ except Exception as e:
272
+ verbose_logger.debug(f"Error parsing message from event: {e}")
273
+ return None
274
+
275
+ def _extract_headers_from_event(self, event) -> InvokeAgentEventHeaders:
276
+ """Extract headers from an AWS event for categorization."""
277
+ try:
278
+ response_dict = event.to_response_dict()
279
+ headers = response_dict.get("headers", {})
280
+
281
+ # Extract the event-type and content-type headers that we care about
282
+ return InvokeAgentEventHeaders(
283
+ event_type=headers.get(":event-type", ""),
284
+ content_type=headers.get(":content-type", ""),
285
+ message_type=headers.get(":message-type", ""),
286
+ )
287
+ except Exception as e:
288
+ verbose_logger.debug(f"Error extracting headers: {e}")
289
+ return InvokeAgentEventHeaders(
290
+ event_type="", content_type="", message_type=""
291
+ )
292
+
293
+ def _get_response_stream_shape(self):
294
+ """Get the response stream shape for parsing, reusing existing logic."""
295
+ try:
296
+ # Try to reuse the cached shape from the existing decoder
297
+ from litellm.llms.bedrock.chat.invoke_handler import (
298
+ get_response_stream_shape,
299
+ )
300
+
301
+ return get_response_stream_shape()
302
+ except ImportError:
303
+ # Fallback: create our own shape
304
+ try:
305
+ from botocore.loaders import Loader
306
+ from botocore.model import ServiceModel
307
+
308
+ loader = Loader()
309
+ bedrock_service_dict = loader.load_service_model(
310
+ "bedrock-runtime", "service-2"
311
+ )
312
+ bedrock_service_model = ServiceModel(bedrock_service_dict)
313
+ return bedrock_service_model.shape_for("ResponseStream")
314
+ except Exception as e:
315
+ verbose_logger.warning(f"Could not load response stream shape: {e}")
316
+ return None
317
+
318
+ def _extract_response_content(self, events: InvokeAgentEventList) -> str:
319
+ """Extract the final response content from parsed events."""
320
+ response_parts = []
321
+
322
+ for event in events:
323
+ headers = event.get("headers", {})
324
+ payload = event.get("payload")
325
+
326
+ event_type = headers.get(
327
+ "event_type"
328
+ ) # Note: using event_type not event-type
329
+
330
+ if event_type == "chunk" and payload:
331
+ # Extract base64 encoded content from chunk events
332
+ chunk_payload: InvokeAgentChunkPayload = payload # type: ignore
333
+ encoded_bytes = chunk_payload.get("bytes", "")
334
+ if encoded_bytes:
335
+ try:
336
+ decoded_content = base64.b64decode(encoded_bytes).decode(
337
+ "utf-8"
338
+ )
339
+ response_parts.append(decoded_content)
340
+ except Exception as e:
341
+ verbose_logger.warning(f"Failed to decode chunk content: {e}")
342
+
343
+ return "".join(response_parts)
344
+
345
+ def _extract_usage_info(self, events: InvokeAgentEventList) -> InvokeAgentUsage:
346
+ """Extract token usage information from trace events."""
347
+ usage_info = InvokeAgentUsage(
348
+ inputTokens=0,
349
+ outputTokens=0,
350
+ model=None,
351
+ )
352
+
353
+ response_model: Optional[str] = None
354
+
355
+ for event in events:
356
+ if not self._is_trace_event(event):
357
+ continue
358
+
359
+ trace_data = self._get_trace_data(event)
360
+ if not trace_data:
361
+ continue
362
+
363
+ verbose_logger.debug(f"Trace event: {trace_data}")
364
+
365
+ # Extract usage from pre-processing trace
366
+ self._extract_and_update_preprocessing_usage(
367
+ trace_data=trace_data,
368
+ usage_info=usage_info,
369
+ )
370
+
371
+ # Extract model from orchestration trace
372
+ if response_model is None:
373
+ response_model = self._extract_orchestration_model(trace_data)
374
+
375
+ usage_info["model"] = response_model
376
+ return usage_info
377
+
378
+ def _is_trace_event(self, event: InvokeAgentEvent) -> bool:
379
+ """Check if the event is a trace event."""
380
+ headers = event.get("headers", {})
381
+ event_type = headers.get("event_type")
382
+ payload = event.get("payload")
383
+ return event_type == "trace" and payload is not None
384
+
385
+ def _get_trace_data(self, event: InvokeAgentEvent) -> Optional[InvokeAgentTrace]:
386
+ """Extract trace data from a trace event."""
387
+ payload = event.get("payload")
388
+ if not payload:
389
+ return None
390
+
391
+ trace_payload: InvokeAgentTracePayload = payload # type: ignore
392
+ return trace_payload.get("trace", {})
393
+
394
+ def _extract_and_update_preprocessing_usage(
395
+ self, trace_data: InvokeAgentTrace, usage_info: InvokeAgentUsage
396
+ ) -> None:
397
+ """Extract usage information from preprocessing trace."""
398
+ pre_processing: Optional[InvokeAgentPreProcessingTrace] = trace_data.get(
399
+ "preProcessingTrace"
400
+ )
401
+ if not pre_processing:
402
+ return
403
+
404
+ model_output: Optional[InvokeAgentModelInvocationOutput] = (
405
+ pre_processing.get("modelInvocationOutput")
406
+ or InvokeAgentModelInvocationOutput()
407
+ )
408
+ if not model_output:
409
+ return
410
+
411
+ metadata: Optional[InvokeAgentMetadata] = (
412
+ model_output.get("metadata") or InvokeAgentMetadata()
413
+ )
414
+ if not metadata:
415
+ return
416
+
417
+ usage: Optional[Union[InvokeAgentUsage, Dict]] = metadata.get("usage", {})
418
+ if not usage:
419
+ return
420
+
421
+ usage_info["inputTokens"] += usage.get("inputTokens", 0)
422
+ usage_info["outputTokens"] += usage.get("outputTokens", 0)
423
+
424
+ def _extract_orchestration_model(
425
+ self, trace_data: InvokeAgentTrace
426
+ ) -> Optional[str]:
427
+ """Extract model information from orchestration trace."""
428
+ orchestration_trace: Optional[InvokeAgentOrchestrationTrace] = trace_data.get(
429
+ "orchestrationTrace"
430
+ )
431
+ if not orchestration_trace:
432
+ return None
433
+
434
+ model_invocation: Optional[InvokeAgentModelInvocationInput] = (
435
+ orchestration_trace.get("modelInvocationInput")
436
+ or InvokeAgentModelInvocationInput()
437
+ )
438
+ if not model_invocation:
439
+ return None
440
+
441
+ return model_invocation.get("foundationModel")
442
+
443
+ def _build_model_response(
444
+ self,
445
+ content: str,
446
+ model: str,
447
+ usage_info: InvokeAgentUsage,
448
+ model_response: ModelResponse,
449
+ ) -> ModelResponse:
450
+ """Build the final ModelResponse object."""
451
+
452
+ # Create the message content
453
+ message = Message(content=content, role="assistant")
454
+
455
+ # Create choices
456
+ choice = Choices(finish_reason="stop", index=0, message=message)
457
+
458
+ # Update model response
459
+ model_response.choices = [choice]
460
+ model_response.model = usage_info.get("model", model)
461
+
462
+ # Add usage information if available
463
+ if usage_info:
464
+ from litellm.types.utils import Usage
465
+
466
+ usage = Usage(
467
+ prompt_tokens=usage_info.get("inputTokens", 0),
468
+ completion_tokens=usage_info.get("outputTokens", 0),
469
+ total_tokens=usage_info.get("inputTokens", 0)
470
+ + usage_info.get("outputTokens", 0),
471
+ )
472
+ setattr(model_response, "usage", usage)
473
+
474
+ return model_response
475
+
476
+ def transform_response(
477
+ self,
478
+ model: str,
479
+ raw_response: httpx.Response,
480
+ model_response: ModelResponse,
481
+ logging_obj: LiteLLMLoggingObj,
482
+ request_data: dict,
483
+ messages: List[AllMessageValues],
484
+ optional_params: dict,
485
+ litellm_params: dict,
486
+ encoding: Any,
487
+ api_key: Optional[str] = None,
488
+ json_mode: Optional[bool] = None,
489
+ ) -> ModelResponse:
490
+ try:
491
+ # Get the raw binary content
492
+ raw_content = raw_response.content
493
+ verbose_logger.debug(
494
+ f"Processing {len(raw_content)} bytes of AWS event stream data"
495
+ )
496
+
497
+ # Parse the AWS event stream format
498
+ events = self._parse_aws_event_stream(raw_content)
499
+ verbose_logger.debug(f"Parsed {len(events)} events from stream")
500
+
501
+ # Extract response content from chunk events
502
+ content = self._extract_response_content(events)
503
+
504
+ # Extract usage information from trace events
505
+ usage_info = self._extract_usage_info(events)
506
+
507
+ # Build and return the model response
508
+ return self._build_model_response(
509
+ content=content,
510
+ model=model,
511
+ usage_info=usage_info,
512
+ model_response=model_response,
513
+ )
514
+
515
+ except Exception as e:
516
+ verbose_logger.error(
517
+ f"Error processing Bedrock Invoke Agent response: {str(e)}"
518
+ )
519
+ raise BedrockError(
520
+ message=f"Error processing response: {str(e)}",
521
+ status_code=raw_response.status_code,
522
+ )
523
+
524
+ def validate_environment(
525
+ self,
526
+ headers: dict,
527
+ model: str,
528
+ messages: List[AllMessageValues],
529
+ optional_params: dict,
530
+ litellm_params: dict,
531
+ api_key: Optional[str] = None,
532
+ api_base: Optional[str] = None,
533
+ ) -> dict:
534
+ return headers
535
+
536
+ def get_error_class(
537
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
538
+ ) -> BaseLLMException:
539
+ return BedrockError(status_code=status_code, message=error_message)
540
+
541
+ def should_fake_stream(
542
+ self,
543
+ model: Optional[str],
544
+ stream: Optional[bool],
545
+ custom_llm_provider: Optional[str] = None,
546
+ ) -> bool:
547
+ return True
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_handler.py ADDED
@@ -0,0 +1,1893 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ TODO: DELETE FILE. Bedrock LLM is no longer used. Goto `litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py`
3
+ """
4
+
5
+ import copy
6
+ import time
7
+ import types
8
+ from functools import partial
9
+ from typing import (
10
+ AsyncIterator,
11
+ Callable,
12
+ Iterator,
13
+ Optional,
14
+ Tuple,
15
+ cast,
16
+ get_args,
17
+ )
18
+
19
+ import httpx # type: ignore
20
+
21
+ import litellm
22
+ from litellm import verbose_logger
23
+ from litellm._uuid import uuid
24
+ from litellm.caching.caching import InMemoryCache
25
+ from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
26
+ from litellm.litellm_core_utils.core_helpers import map_finish_reason
27
+ from litellm.litellm_core_utils.litellm_logging import Logging
28
+ from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
29
+ from litellm.litellm_core_utils.prompt_templates.factory import (
30
+ cohere_message_pt,
31
+ construct_tool_use_system_prompt,
32
+ contains_tag,
33
+ custom_prompt,
34
+ extract_between_tags,
35
+ parse_xml_params,
36
+ prompt_factory,
37
+ )
38
+ from litellm.llms.anthropic.chat.handler import (
39
+ ModelResponseIterator as AnthropicModelResponseIterator,
40
+ )
41
+ from litellm.llms.custom_httpx.http_handler import (
42
+ AsyncHTTPHandler,
43
+ HTTPHandler,
44
+ _get_httpx_client,
45
+ get_async_httpx_client,
46
+ )
47
+ from litellm.types.llms.bedrock import *
48
+ from litellm.types.llms.openai import (
49
+ ChatCompletionRedactedThinkingBlock,
50
+ ChatCompletionThinkingBlock,
51
+ ChatCompletionToolCallChunk,
52
+ ChatCompletionToolCallFunctionChunk,
53
+ ChatCompletionUsageBlock,
54
+ )
55
+ from litellm.types.utils import (
56
+ ChatCompletionMessageToolCall,
57
+ Choices,
58
+ Delta,
59
+ )
60
+ from litellm.types.utils import GenericStreamingChunk as GChunk
61
+ from litellm.types.utils import (
62
+ ModelResponse,
63
+ ModelResponseStream,
64
+ StreamingChoices,
65
+ Usage,
66
+ )
67
+ from litellm.utils import CustomStreamWrapper, get_secret
68
+
69
+ from ..base_aws_llm import BaseAWSLLM
70
+ from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name
71
+
72
+ _response_stream_shape_cache = None
73
+ bedrock_tool_name_mappings: InMemoryCache = InMemoryCache(
74
+ max_size_in_memory=50, default_ttl=600
75
+ )
76
+ from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
77
+ from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
78
+ AmazonBedrockOpenAIConfig,
79
+ )
80
+
81
+ converse_config = AmazonConverseConfig()
82
+
83
+
84
+ class AmazonCohereChatConfig:
85
+ """
86
+ Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command-r-plus.html
87
+ """
88
+
89
+ documents: Optional[List[Document]] = None
90
+ search_queries_only: Optional[bool] = None
91
+ preamble: Optional[str] = None
92
+ max_tokens: Optional[int] = None
93
+ temperature: Optional[float] = None
94
+ p: Optional[float] = None
95
+ k: Optional[float] = None
96
+ prompt_truncation: Optional[str] = None
97
+ frequency_penalty: Optional[float] = None
98
+ presence_penalty: Optional[float] = None
99
+ seed: Optional[int] = None
100
+ return_prompt: Optional[bool] = None
101
+ stop_sequences: Optional[List[str]] = None
102
+ raw_prompting: Optional[bool] = None
103
+
104
+ def __init__(
105
+ self,
106
+ documents: Optional[List[Document]] = None,
107
+ search_queries_only: Optional[bool] = None,
108
+ preamble: Optional[str] = None,
109
+ max_tokens: Optional[int] = None,
110
+ temperature: Optional[float] = None,
111
+ p: Optional[float] = None,
112
+ k: Optional[float] = None,
113
+ prompt_truncation: Optional[str] = None,
114
+ frequency_penalty: Optional[float] = None,
115
+ presence_penalty: Optional[float] = None,
116
+ seed: Optional[int] = None,
117
+ return_prompt: Optional[bool] = None,
118
+ stop_sequences: Optional[str] = None,
119
+ raw_prompting: Optional[bool] = None,
120
+ ) -> None:
121
+ locals_ = locals().copy()
122
+ for key, value in locals_.items():
123
+ if key != "self" and value is not None:
124
+ setattr(self.__class__, key, value)
125
+
126
+ @classmethod
127
+ def get_config(cls):
128
+ return {
129
+ k: v
130
+ for k, v in cls.__dict__.items()
131
+ if not k.startswith("__")
132
+ and not isinstance(
133
+ v,
134
+ (
135
+ types.FunctionType,
136
+ types.BuiltinFunctionType,
137
+ classmethod,
138
+ staticmethod,
139
+ ),
140
+ )
141
+ and v is not None
142
+ }
143
+
144
+ def get_supported_openai_params(self) -> List[str]:
145
+ return [
146
+ "max_tokens",
147
+ "max_completion_tokens",
148
+ "stream",
149
+ "stop",
150
+ "temperature",
151
+ "top_p",
152
+ "frequency_penalty",
153
+ "presence_penalty",
154
+ "seed",
155
+ "stop",
156
+ "tools",
157
+ "tool_choice",
158
+ ]
159
+
160
+ def map_openai_params(
161
+ self, non_default_params: dict, optional_params: dict
162
+ ) -> dict:
163
+ for param, value in non_default_params.items():
164
+ if param == "max_tokens" or param == "max_completion_tokens":
165
+ optional_params["max_tokens"] = value
166
+ if param == "stream":
167
+ optional_params["stream"] = value
168
+ if param == "stop":
169
+ if isinstance(value, str):
170
+ value = [value]
171
+ optional_params["stop_sequences"] = value
172
+ if param == "temperature":
173
+ optional_params["temperature"] = value
174
+ if param == "top_p":
175
+ optional_params["p"] = value
176
+ if param == "frequency_penalty":
177
+ optional_params["frequency_penalty"] = value
178
+ if param == "presence_penalty":
179
+ optional_params["presence_penalty"] = value
180
+ if "seed":
181
+ optional_params["seed"] = value
182
+ return optional_params
183
+
184
+
185
+ async def make_call(
186
+ client: Optional[AsyncHTTPHandler],
187
+ api_base: str,
188
+ headers: dict,
189
+ data: str,
190
+ model: str,
191
+ messages: list,
192
+ logging_obj: Logging,
193
+ fake_stream: bool = False,
194
+ json_mode: Optional[bool] = False,
195
+ bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
196
+ stream_chunk_size: int = 1024,
197
+ ):
198
+ try:
199
+ if client is None:
200
+ client = get_async_httpx_client(
201
+ llm_provider=litellm.LlmProviders.BEDROCK,
202
+ params={"ssl_verify": logging_obj.litellm_params.get("ssl_verify")}
203
+ if logging_obj
204
+ and logging_obj.litellm_params
205
+ and logging_obj.litellm_params.get("ssl_verify")
206
+ else None,
207
+ ) # Create a new client if none provided
208
+
209
+ response = await client.post(
210
+ api_base,
211
+ headers=headers,
212
+ data=data,
213
+ stream=not fake_stream,
214
+ logging_obj=logging_obj,
215
+ )
216
+
217
+ if response.status_code != 200:
218
+ raise BedrockError(status_code=response.status_code, message=response.text)
219
+
220
+ if fake_stream:
221
+ model_response: (
222
+ ModelResponse
223
+ ) = litellm.AmazonConverseConfig()._transform_response(
224
+ model=model,
225
+ response=response,
226
+ model_response=litellm.ModelResponse(),
227
+ stream=True,
228
+ logging_obj=logging_obj,
229
+ optional_params={},
230
+ api_key="",
231
+ data=data,
232
+ messages=messages,
233
+ encoding=litellm.encoding,
234
+ ) # type: ignore
235
+ completion_stream: Any = MockResponseIterator(
236
+ model_response=model_response, json_mode=json_mode
237
+ )
238
+ elif bedrock_invoke_provider == "anthropic":
239
+ decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder(
240
+ model=model,
241
+ sync_stream=False,
242
+ json_mode=json_mode,
243
+ )
244
+ completion_stream = decoder.aiter_bytes(
245
+ response.aiter_bytes(chunk_size=stream_chunk_size)
246
+ )
247
+ elif bedrock_invoke_provider == "deepseek_r1":
248
+ decoder = AmazonDeepSeekR1StreamDecoder(
249
+ model=model,
250
+ sync_stream=False,
251
+ )
252
+ completion_stream = decoder.aiter_bytes(
253
+ response.aiter_bytes(chunk_size=stream_chunk_size)
254
+ )
255
+ else:
256
+ decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
257
+ completion_stream = decoder.aiter_bytes(
258
+ response.aiter_bytes(chunk_size=stream_chunk_size)
259
+ )
260
+
261
+ # LOGGING
262
+ logging_obj.post_call(
263
+ input=messages,
264
+ api_key="",
265
+ original_response="first stream response received",
266
+ additional_args={"complete_input_dict": data},
267
+ )
268
+
269
+ return completion_stream
270
+ except httpx.HTTPStatusError as err:
271
+ error_code = err.response.status_code
272
+ raise BedrockError(status_code=error_code, message=err.response.text)
273
+ except httpx.TimeoutException:
274
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
275
+ except Exception as e:
276
+ raise BedrockError(status_code=500, message=str(e))
277
+
278
+
279
+ def make_sync_call(
280
+ client: Optional[HTTPHandler],
281
+ api_base: str,
282
+ headers: dict,
283
+ data: str,
284
+ signed_json_body: Optional[bytes],
285
+ model: str,
286
+ messages: list,
287
+ logging_obj: Logging,
288
+ fake_stream: bool = False,
289
+ json_mode: Optional[bool] = False,
290
+ bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
291
+ stream_chunk_size: int = 1024,
292
+ ):
293
+ try:
294
+ if client is None:
295
+ client = _get_httpx_client(
296
+ params={"ssl_verify": logging_obj.litellm_params.get("ssl_verify")}
297
+ if logging_obj
298
+ and logging_obj.litellm_params
299
+ and logging_obj.litellm_params.get("ssl_verify")
300
+ else None
301
+ )
302
+
303
+ response = client.post(
304
+ api_base,
305
+ headers=headers,
306
+ data=signed_json_body if signed_json_body is not None else data,
307
+ stream=not fake_stream,
308
+ logging_obj=logging_obj,
309
+ )
310
+
311
+ if response.status_code != 200:
312
+ raise BedrockError(status_code=response.status_code, message=response.text)
313
+
314
+ if fake_stream:
315
+ model_response: (
316
+ ModelResponse
317
+ ) = litellm.AmazonConverseConfig()._transform_response(
318
+ model=model,
319
+ response=response,
320
+ model_response=litellm.ModelResponse(),
321
+ stream=True,
322
+ logging_obj=logging_obj,
323
+ optional_params={},
324
+ api_key="",
325
+ data=data,
326
+ messages=messages,
327
+ encoding=litellm.encoding,
328
+ ) # type: ignore
329
+ completion_stream: Any = MockResponseIterator(
330
+ model_response=model_response, json_mode=json_mode
331
+ )
332
+ elif bedrock_invoke_provider == "anthropic":
333
+ decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder(
334
+ model=model,
335
+ sync_stream=True,
336
+ json_mode=json_mode,
337
+ )
338
+ completion_stream = decoder.iter_bytes(
339
+ response.iter_bytes(chunk_size=stream_chunk_size)
340
+ )
341
+ elif bedrock_invoke_provider == "deepseek_r1":
342
+ decoder = AmazonDeepSeekR1StreamDecoder(
343
+ model=model,
344
+ sync_stream=True,
345
+ )
346
+ completion_stream = decoder.iter_bytes(
347
+ response.iter_bytes(chunk_size=stream_chunk_size)
348
+ )
349
+ else:
350
+ decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
351
+ completion_stream = decoder.iter_bytes(
352
+ response.iter_bytes(chunk_size=stream_chunk_size)
353
+ )
354
+
355
+ # LOGGING
356
+ logging_obj.post_call(
357
+ input=messages,
358
+ api_key="",
359
+ original_response="first stream response received",
360
+ additional_args={"complete_input_dict": data},
361
+ )
362
+
363
+ return completion_stream
364
+ except httpx.HTTPStatusError as err:
365
+ error_code = err.response.status_code
366
+ raise BedrockError(status_code=error_code, message=err.response.text)
367
+ except httpx.TimeoutException:
368
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
369
+ except Exception as e:
370
+ raise BedrockError(status_code=500, message=str(e))
371
+
372
+
373
+ class BedrockLLM(BaseAWSLLM):
374
+ """
375
+ Example call
376
+
377
+ ```
378
+ curl --location --request POST 'https://bedrock-runtime.{aws_region_name}.amazonaws.com/model/{bedrock_model_name}/invoke' \
379
+ --header 'Content-Type: application/json' \
380
+ --header 'Accept: application/json' \
381
+ --user "$AWS_ACCESS_KEY_ID":"$AWS_SECRET_ACCESS_KEY" \
382
+ --aws-sigv4 "aws:amz:us-east-1:bedrock" \
383
+ --data-raw '{
384
+ "prompt": "Hi",
385
+ "temperature": 0,
386
+ "p": 0.9,
387
+ "max_tokens": 4096
388
+ }'
389
+ ```
390
+ """
391
+
392
+ def __init__(self) -> None:
393
+ super().__init__()
394
+
395
+ @staticmethod
396
+ def is_claude_messages_api_model(model: str) -> bool:
397
+ """
398
+ Check if the model uses the Claude Messages API (Claude 3+).
399
+
400
+ Handles:
401
+ - Regional prefixes: eu.anthropic.claude-*, us.anthropic.claude-*
402
+ - Claude 3 models: claude-3-haiku, claude-3-sonnet, claude-3-opus, claude-3-5-*, claude-3-7-*
403
+ - Claude 4 models: claude-opus-4, claude-sonnet-4, claude-haiku-4
404
+ """
405
+ # Normalize model string to lowercase for matching
406
+ model_lower = model.lower()
407
+
408
+ # Claude 3+ indicators (all use Messages API)
409
+ messages_api_indicators = [
410
+ "claude-3", # Claude 3.x models
411
+ "claude-opus-4", # Claude Opus 4
412
+ "claude-sonnet-4", # Claude Sonnet 4
413
+ "claude-haiku-4", # Claude Haiku 4
414
+ ]
415
+
416
+ return any(indicator in model_lower for indicator in messages_api_indicators)
417
+
418
+ def convert_messages_to_prompt(
419
+ self, model, messages, provider, custom_prompt_dict
420
+ ) -> Tuple[str, Optional[list]]:
421
+ # handle anthropic prompts and amazon titan prompts
422
+ prompt = ""
423
+ chat_history: Optional[list] = None
424
+ ## CUSTOM PROMPT
425
+ if model in custom_prompt_dict:
426
+ # check if the model has a registered custom prompt
427
+ model_prompt_details = custom_prompt_dict[model]
428
+ prompt = custom_prompt(
429
+ role_dict=model_prompt_details["roles"],
430
+ initial_prompt_value=model_prompt_details.get(
431
+ "initial_prompt_value", ""
432
+ ),
433
+ final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
434
+ messages=messages,
435
+ )
436
+ return prompt, None
437
+ ## ELSE
438
+ if provider == "anthropic" or provider == "amazon":
439
+ prompt = prompt_factory(
440
+ model=model, messages=messages, custom_llm_provider="bedrock"
441
+ )
442
+ elif provider == "mistral":
443
+ prompt = prompt_factory(
444
+ model=model, messages=messages, custom_llm_provider="bedrock"
445
+ )
446
+ elif provider == "meta" or provider == "llama":
447
+ prompt = prompt_factory(
448
+ model=model, messages=messages, custom_llm_provider="bedrock"
449
+ )
450
+ elif provider == "openai":
451
+ # OpenAI uses messages directly, no prompt conversion needed
452
+ # Return empty prompt as it won't be used
453
+ prompt = ""
454
+ elif provider == "cohere":
455
+ prompt, chat_history = cohere_message_pt(messages=messages)
456
+ else:
457
+ prompt = ""
458
+ for message in messages:
459
+ if "role" in message:
460
+ if message["role"] == "user":
461
+ prompt += f"{message['content']}"
462
+ else:
463
+ prompt += f"{message['content']}"
464
+ else:
465
+ prompt += f"{message['content']}"
466
+ return prompt, chat_history # type: ignore
467
+
468
+ def process_response( # noqa: PLR0915
469
+ self,
470
+ model: str,
471
+ response: httpx.Response,
472
+ model_response: ModelResponse,
473
+ stream: Optional[bool],
474
+ logging_obj: Logging,
475
+ optional_params: dict,
476
+ api_key: str,
477
+ data: Union[dict, str],
478
+ messages: List,
479
+ print_verbose,
480
+ encoding,
481
+ ) -> Union[ModelResponse, CustomStreamWrapper]:
482
+ provider = self.get_bedrock_invoke_provider(model)
483
+ ## LOGGING
484
+ logging_obj.post_call(
485
+ input=messages,
486
+ api_key=api_key,
487
+ original_response=response.text,
488
+ additional_args={"complete_input_dict": data},
489
+ )
490
+ print_verbose(f"raw model_response: {response.text}")
491
+
492
+ ## RESPONSE OBJECT
493
+ try:
494
+ completion_response = response.json()
495
+ except Exception:
496
+ raise BedrockError(message=response.text, status_code=422)
497
+
498
+ outputText: Optional[str] = None
499
+ try:
500
+ if provider == "cohere":
501
+ if "text" in completion_response:
502
+ outputText = completion_response["text"] # type: ignore
503
+ elif "generations" in completion_response:
504
+ outputText = completion_response["generations"][0]["text"]
505
+ model_response.choices[0].finish_reason = map_finish_reason(
506
+ completion_response["generations"][0]["finish_reason"]
507
+ )
508
+ elif provider == "anthropic":
509
+ if self.is_claude_messages_api_model(model):
510
+ json_schemas: dict = {}
511
+ _is_function_call = False
512
+ ## Handle Tool Calling
513
+ if "tools" in optional_params:
514
+ _is_function_call = True
515
+ for tool in optional_params["tools"]:
516
+ json_schemas[tool["function"]["name"]] = tool[
517
+ "function"
518
+ ].get("parameters", None)
519
+ outputText = completion_response.get("content")[0].get("text", None)
520
+ if outputText is not None and contains_tag(
521
+ "invoke", outputText
522
+ ): # OUTPUT PARSE FUNCTION CALL
523
+ function_name = extract_between_tags("tool_name", outputText)[0]
524
+ function_arguments_str = extract_between_tags(
525
+ "invoke", outputText
526
+ )[0].strip()
527
+ function_arguments_str = (
528
+ f"<invoke>{function_arguments_str}</invoke>"
529
+ )
530
+ function_arguments = parse_xml_params(
531
+ function_arguments_str,
532
+ json_schema=json_schemas.get(
533
+ function_name, None
534
+ ), # check if we have a json schema for this function name)
535
+ )
536
+ _message = litellm.Message(
537
+ tool_calls=[
538
+ {
539
+ "id": f"call_{uuid.uuid4()}",
540
+ "type": "function",
541
+ "function": {
542
+ "name": function_name,
543
+ "arguments": json.dumps(function_arguments),
544
+ },
545
+ }
546
+ ],
547
+ content=None,
548
+ )
549
+ model_response.choices[0].message = _message # type: ignore
550
+ model_response._hidden_params[
551
+ "original_response"
552
+ ] = outputText # allow user to access raw anthropic tool calling response
553
+ if (
554
+ _is_function_call is True
555
+ and stream is not None
556
+ and stream is True
557
+ ):
558
+ print_verbose(
559
+ "INSIDE BEDROCK STREAMING TOOL CALLING CONDITION BLOCK"
560
+ )
561
+ # return an iterator
562
+ streaming_model_response = ModelResponseStream()
563
+ streaming_model_response.choices[0].finish_reason = getattr(
564
+ model_response.choices[0], "finish_reason", "stop"
565
+ )
566
+ # streaming_model_response.choices = [litellm.utils.StreamingChoices()]
567
+ streaming_choice = litellm.utils.StreamingChoices()
568
+ streaming_choice.index = model_response.choices[0].index
569
+ _tool_calls = []
570
+ print_verbose(
571
+ f"type of model_response.choices[0]: {type(model_response.choices[0])}"
572
+ )
573
+ print_verbose(
574
+ f"type of streaming_choice: {type(streaming_choice)}"
575
+ )
576
+ if isinstance(model_response.choices[0], litellm.Choices):
577
+ if getattr(
578
+ model_response.choices[0].message, "tool_calls", None
579
+ ) is not None and isinstance(
580
+ model_response.choices[0].message.tool_calls, list
581
+ ):
582
+ for tool_call in model_response.choices[
583
+ 0
584
+ ].message.tool_calls:
585
+ _tool_call = {**tool_call.dict(), "index": 0}
586
+ _tool_calls.append(_tool_call)
587
+ delta_obj = Delta(
588
+ content=getattr(
589
+ model_response.choices[0].message, "content", None
590
+ ),
591
+ role=model_response.choices[0].message.role,
592
+ tool_calls=_tool_calls,
593
+ )
594
+ streaming_choice.delta = delta_obj
595
+ streaming_model_response.choices = [streaming_choice]
596
+ completion_stream = ModelResponseIterator(
597
+ model_response=streaming_model_response
598
+ )
599
+ print_verbose(
600
+ "Returns anthropic CustomStreamWrapper with 'cached_response' streaming object"
601
+ )
602
+ return litellm.CustomStreamWrapper(
603
+ completion_stream=completion_stream,
604
+ model=model,
605
+ custom_llm_provider="cached_response",
606
+ logging_obj=logging_obj,
607
+ )
608
+
609
+ model_response.choices[0].finish_reason = map_finish_reason(
610
+ completion_response.get("stop_reason", "")
611
+ )
612
+ _usage = litellm.Usage(
613
+ prompt_tokens=completion_response["usage"]["input_tokens"],
614
+ completion_tokens=completion_response["usage"]["output_tokens"],
615
+ total_tokens=completion_response["usage"]["input_tokens"]
616
+ + completion_response["usage"]["output_tokens"],
617
+ )
618
+ setattr(model_response, "usage", _usage)
619
+ else:
620
+ outputText = completion_response["completion"]
621
+
622
+ model_response.choices[0].finish_reason = completion_response[
623
+ "stop_reason"
624
+ ]
625
+ elif provider == "ai21":
626
+ outputText = (
627
+ completion_response.get("completions")[0].get("data").get("text")
628
+ )
629
+ elif provider == "meta" or provider == "llama":
630
+ outputText = completion_response["generation"]
631
+ elif provider == "openai":
632
+ # OpenAI imported models use OpenAI Chat Completions format
633
+ if (
634
+ "choices" in completion_response
635
+ and len(completion_response["choices"]) > 0
636
+ ):
637
+ choice = completion_response["choices"][0]
638
+ if "message" in choice:
639
+ outputText = choice["message"].get("content")
640
+ elif "text" in choice: # fallback for completion format
641
+ outputText = choice["text"]
642
+
643
+ # Set finish reason
644
+ if "finish_reason" in choice:
645
+ model_response.choices[0].finish_reason = map_finish_reason(
646
+ choice["finish_reason"]
647
+ )
648
+
649
+ # Set usage if available
650
+ if "usage" in completion_response:
651
+ usage = completion_response["usage"]
652
+ _usage = litellm.Usage(
653
+ prompt_tokens=usage.get("prompt_tokens", 0),
654
+ completion_tokens=usage.get("completion_tokens", 0),
655
+ total_tokens=usage.get("total_tokens", 0),
656
+ )
657
+ setattr(model_response, "usage", _usage)
658
+ elif provider == "mistral":
659
+ outputText = completion_response["outputs"][0]["text"]
660
+ model_response.choices[0].finish_reason = completion_response[
661
+ "outputs"
662
+ ][0]["stop_reason"]
663
+ else: # amazon titan
664
+ outputText = completion_response.get("results")[0].get("outputText")
665
+ except Exception as e:
666
+ raise BedrockError(
667
+ message="Error processing={}, Received error={}".format(
668
+ response.text, str(e)
669
+ ),
670
+ status_code=422,
671
+ )
672
+
673
+ try:
674
+ if (
675
+ outputText is not None
676
+ and len(outputText) > 0
677
+ and hasattr(model_response.choices[0], "message")
678
+ and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore
679
+ is None
680
+ ):
681
+ model_response.choices[0].message.content = outputText # type: ignore
682
+ elif (
683
+ hasattr(model_response.choices[0], "message")
684
+ and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore
685
+ is not None
686
+ ):
687
+ pass
688
+ else:
689
+ raise Exception()
690
+ except Exception as e:
691
+ raise BedrockError(
692
+ message="Error parsing received text={}.\nError-{}".format(
693
+ outputText, str(e)
694
+ ),
695
+ status_code=response.status_code,
696
+ )
697
+
698
+ if stream and provider == "ai21":
699
+ streaming_model_response = ModelResponseStream()
700
+ streaming_model_response.choices[0].finish_reason = model_response.choices[ # type: ignore
701
+ 0
702
+ ].finish_reason
703
+ # streaming_model_response.choices = [litellm.utils.StreamingChoices()]
704
+ streaming_choice = litellm.utils.StreamingChoices()
705
+ streaming_choice.index = model_response.choices[0].index
706
+ delta_obj = litellm.utils.Delta(
707
+ content=getattr(model_response.choices[0].message, "content", None), # type: ignore
708
+ role=model_response.choices[0].message.role, # type: ignore
709
+ )
710
+ streaming_choice.delta = delta_obj
711
+ streaming_model_response.choices = [streaming_choice]
712
+ mri = ModelResponseIterator(model_response=streaming_model_response)
713
+ return CustomStreamWrapper(
714
+ completion_stream=mri,
715
+ model=model,
716
+ custom_llm_provider="cached_response",
717
+ logging_obj=logging_obj,
718
+ )
719
+
720
+ ## CALCULATING USAGE - bedrock returns usage in the headers
721
+ # Skip if usage was already set (e.g., from JSON response for OpenAI provider)
722
+ if (
723
+ not hasattr(model_response, "usage")
724
+ or getattr(model_response, "usage", None) is None
725
+ ):
726
+ bedrock_input_tokens = response.headers.get(
727
+ "x-amzn-bedrock-input-token-count", None
728
+ )
729
+ bedrock_output_tokens = response.headers.get(
730
+ "x-amzn-bedrock-output-token-count", None
731
+ )
732
+
733
+ prompt_tokens = int(
734
+ bedrock_input_tokens or litellm.token_counter(messages=messages)
735
+ )
736
+
737
+ completion_tokens = int(
738
+ bedrock_output_tokens
739
+ or litellm.token_counter(
740
+ text=model_response.choices[0].message.content, # type: ignore
741
+ count_response_tokens=True,
742
+ )
743
+ )
744
+
745
+ model_response.created = int(time.time())
746
+ model_response.model = model
747
+ usage = Usage(
748
+ prompt_tokens=prompt_tokens,
749
+ completion_tokens=completion_tokens,
750
+ total_tokens=prompt_tokens + completion_tokens,
751
+ )
752
+ setattr(model_response, "usage", usage)
753
+ else:
754
+ # Ensure created and model are set even if usage was already set
755
+ model_response.created = int(time.time())
756
+ model_response.model = model
757
+
758
+ return model_response
759
+
760
+ def completion( # noqa: PLR0915
761
+ self,
762
+ model: str,
763
+ messages: list,
764
+ api_base: Optional[str],
765
+ custom_prompt_dict: dict,
766
+ model_response: ModelResponse,
767
+ print_verbose: Callable,
768
+ encoding,
769
+ logging_obj: Logging,
770
+ optional_params: dict,
771
+ acompletion: bool,
772
+ timeout: Optional[Union[float, httpx.Timeout]],
773
+ litellm_params=None,
774
+ logger_fn=None,
775
+ extra_headers: Optional[dict] = None,
776
+ client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None,
777
+ ) -> Union[ModelResponse, CustomStreamWrapper]:
778
+ try:
779
+ from botocore.credentials import Credentials
780
+ except ImportError:
781
+ raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
782
+
783
+ ## SETUP ##
784
+ stream = optional_params.pop("stream", None)
785
+ stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
786
+
787
+ provider = self.get_bedrock_invoke_provider(model)
788
+ modelId = self.get_bedrock_model_id(
789
+ model=model,
790
+ provider=provider,
791
+ optional_params=optional_params,
792
+ )
793
+
794
+ ## CREDENTIALS ##
795
+ # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
796
+ aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
797
+ aws_access_key_id = optional_params.pop("aws_access_key_id", None)
798
+ aws_session_token = optional_params.pop("aws_session_token", None)
799
+ aws_region_name = optional_params.pop("aws_region_name", None)
800
+ aws_role_name = optional_params.pop("aws_role_name", None)
801
+ aws_session_name = optional_params.pop("aws_session_name", None)
802
+ aws_profile_name = optional_params.pop("aws_profile_name", None)
803
+ aws_bedrock_runtime_endpoint = optional_params.pop(
804
+ "aws_bedrock_runtime_endpoint", None
805
+ ) # https://bedrock-runtime.{region_name}.amazonaws.com
806
+ aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
807
+ aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
808
+ ssl_verify = optional_params.pop("ssl_verify", None)
809
+
810
+ ### SET REGION NAME ###
811
+ if aws_region_name is None:
812
+ # check env #
813
+ litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
814
+
815
+ if litellm_aws_region_name is not None and isinstance(
816
+ litellm_aws_region_name, str
817
+ ):
818
+ aws_region_name = litellm_aws_region_name
819
+
820
+ standard_aws_region_name = get_secret("AWS_REGION", None)
821
+ if standard_aws_region_name is not None and isinstance(
822
+ standard_aws_region_name, str
823
+ ):
824
+ aws_region_name = standard_aws_region_name
825
+
826
+ if aws_region_name is None:
827
+ aws_region_name = "us-west-2"
828
+
829
+ credentials: Credentials = self.get_credentials(
830
+ aws_access_key_id=aws_access_key_id,
831
+ aws_secret_access_key=aws_secret_access_key,
832
+ aws_session_token=aws_session_token,
833
+ aws_region_name=aws_region_name,
834
+ aws_session_name=aws_session_name,
835
+ aws_profile_name=aws_profile_name,
836
+ aws_role_name=aws_role_name,
837
+ aws_web_identity_token=aws_web_identity_token,
838
+ aws_sts_endpoint=aws_sts_endpoint,
839
+ ssl_verify=ssl_verify,
840
+ )
841
+
842
+ ### SET RUNTIME ENDPOINT ###
843
+ endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
844
+ api_base=api_base,
845
+ aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
846
+ aws_region_name=aws_region_name,
847
+ )
848
+
849
+ if (stream is not None and stream is True) and provider != "ai21":
850
+ endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream"
851
+ proxy_endpoint_url = (
852
+ f"{proxy_endpoint_url}/model/{modelId}/invoke-with-response-stream"
853
+ )
854
+ else:
855
+ endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
856
+ proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
857
+
858
+ prompt, chat_history = self.convert_messages_to_prompt(
859
+ model, messages, provider, custom_prompt_dict
860
+ )
861
+ inference_params = copy.deepcopy(optional_params)
862
+ json_schemas: dict = {}
863
+ if provider == "cohere":
864
+ if model.startswith("cohere.command-r"):
865
+ ## LOAD CONFIG
866
+ config = litellm.AmazonCohereChatConfig().get_config()
867
+ for k, v in config.items():
868
+ if (
869
+ k not in inference_params
870
+ ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
871
+ inference_params[k] = v
872
+ _data = {"message": prompt, **inference_params}
873
+ if chat_history is not None:
874
+ _data["chat_history"] = chat_history
875
+ data = json.dumps(_data)
876
+ else:
877
+ ## LOAD CONFIG
878
+ config = litellm.AmazonCohereConfig.get_config()
879
+ for k, v in config.items():
880
+ if (
881
+ k not in inference_params
882
+ ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
883
+ inference_params[k] = v
884
+ if stream is True:
885
+ inference_params[
886
+ "stream"
887
+ ] = True # cohere requires stream = True in inference params
888
+ data = json.dumps({"prompt": prompt, **inference_params})
889
+ elif provider == "anthropic":
890
+ if self.is_claude_messages_api_model(model):
891
+ # Separate system prompt from rest of message
892
+ system_prompt_idx: list[int] = []
893
+ system_messages: list[str] = []
894
+ for idx, message in enumerate(messages):
895
+ if message["role"] == "system":
896
+ system_messages.append(message["content"])
897
+ system_prompt_idx.append(idx)
898
+ if len(system_prompt_idx) > 0:
899
+ inference_params["system"] = "\n".join(system_messages)
900
+ messages = [
901
+ i for j, i in enumerate(messages) if j not in system_prompt_idx
902
+ ]
903
+ # Format rest of message according to anthropic guidelines
904
+ messages = prompt_factory(
905
+ model=model, messages=messages, custom_llm_provider="anthropic_xml"
906
+ ) # type: ignore
907
+ ## LOAD CONFIG
908
+ config = litellm.AmazonAnthropicClaudeConfig.get_config()
909
+ for k, v in config.items():
910
+ if (
911
+ k not in inference_params
912
+ ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
913
+ inference_params[k] = v
914
+ ## Handle Tool Calling
915
+ if "tools" in inference_params:
916
+ _is_function_call = True
917
+ for tool in inference_params["tools"]:
918
+ json_schemas[tool["function"]["name"]] = tool["function"].get(
919
+ "parameters", None
920
+ )
921
+ tool_calling_system_prompt = construct_tool_use_system_prompt(
922
+ tools=inference_params["tools"]
923
+ )
924
+ inference_params["system"] = (
925
+ inference_params.get("system", "\n")
926
+ + tool_calling_system_prompt
927
+ ) # add the anthropic tool calling prompt to the system prompt
928
+ inference_params.pop("tools")
929
+ data = json.dumps({"messages": messages, **inference_params})
930
+ else:
931
+ ## LOAD CONFIG
932
+ config = litellm.AmazonAnthropicConfig.get_config()
933
+ for k, v in config.items():
934
+ if (
935
+ k not in inference_params
936
+ ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
937
+ inference_params[k] = v
938
+ data = json.dumps({"prompt": prompt, **inference_params})
939
+ elif provider == "ai21":
940
+ ## LOAD CONFIG
941
+ config = litellm.AmazonAI21Config.get_config()
942
+ for k, v in config.items():
943
+ if (
944
+ k not in inference_params
945
+ ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
946
+ inference_params[k] = v
947
+
948
+ data = json.dumps({"prompt": prompt, **inference_params})
949
+ elif provider == "mistral":
950
+ ## LOAD CONFIG
951
+ config = litellm.AmazonMistralConfig.get_config()
952
+ for k, v in config.items():
953
+ if (
954
+ k not in inference_params
955
+ ): # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in
956
+ inference_params[k] = v
957
+
958
+ data = json.dumps({"prompt": prompt, **inference_params})
959
+ elif provider == "amazon": # amazon titan
960
+ ## LOAD CONFIG
961
+ config = litellm.AmazonTitanConfig.get_config()
962
+ for k, v in config.items():
963
+ if (
964
+ k not in inference_params
965
+ ): # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in
966
+ inference_params[k] = v
967
+
968
+ data = json.dumps(
969
+ {
970
+ "inputText": prompt,
971
+ "textGenerationConfig": inference_params,
972
+ }
973
+ )
974
+ elif provider == "meta" or provider == "llama":
975
+ ## LOAD CONFIG
976
+ config = litellm.AmazonLlamaConfig.get_config()
977
+ for k, v in config.items():
978
+ if (
979
+ k not in inference_params
980
+ ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
981
+ inference_params[k] = v
982
+ data = json.dumps({"prompt": prompt, **inference_params})
983
+ elif provider == "openai":
984
+ ## OpenAI imported models use OpenAI Chat Completions format (messages-based)
985
+ # Use AmazonBedrockOpenAIConfig for proper OpenAI transformation
986
+ openai_config = AmazonBedrockOpenAIConfig()
987
+ supported_params = openai_config.get_supported_openai_params(model=model)
988
+
989
+ # Filter to only supported OpenAI params
990
+ filtered_params = {
991
+ k: v for k, v in inference_params.items() if k in supported_params
992
+ }
993
+
994
+ # OpenAI uses messages format, not prompt
995
+ data = json.dumps({"messages": messages, **filtered_params})
996
+ else:
997
+ ## LOGGING
998
+ logging_obj.pre_call(
999
+ input=messages,
1000
+ api_key="",
1001
+ additional_args={
1002
+ "complete_input_dict": inference_params,
1003
+ },
1004
+ )
1005
+ raise BedrockError(
1006
+ status_code=404,
1007
+ message="Bedrock Invoke HTTPX: Unknown provider={}, model={}. Try calling via converse route - `bedrock/converse/<model>`.".format(
1008
+ provider, model
1009
+ ),
1010
+ )
1011
+
1012
+ ## COMPLETION CALL
1013
+
1014
+ headers = {"Content-Type": "application/json"}
1015
+ if extra_headers is not None:
1016
+ headers = {"Content-Type": "application/json", **extra_headers}
1017
+ prepped = self.get_request_headers(
1018
+ credentials=credentials,
1019
+ aws_region_name=aws_region_name,
1020
+ extra_headers=extra_headers,
1021
+ endpoint_url=endpoint_url,
1022
+ data=data,
1023
+ headers=headers,
1024
+ )
1025
+
1026
+ ## LOGGING
1027
+ logging_obj.pre_call(
1028
+ input=messages,
1029
+ api_key="",
1030
+ additional_args={
1031
+ "complete_input_dict": data,
1032
+ "api_base": proxy_endpoint_url,
1033
+ "headers": prepped.headers,
1034
+ },
1035
+ )
1036
+
1037
+ ### ROUTING (ASYNC, STREAMING, SYNC)
1038
+ if acompletion:
1039
+ if isinstance(client, HTTPHandler):
1040
+ client = None
1041
+ if stream is True and provider != "ai21":
1042
+ return self.async_streaming(
1043
+ model=model,
1044
+ messages=messages,
1045
+ data=data,
1046
+ api_base=proxy_endpoint_url,
1047
+ model_response=model_response,
1048
+ print_verbose=print_verbose,
1049
+ encoding=encoding,
1050
+ logging_obj=logging_obj,
1051
+ optional_params=optional_params,
1052
+ stream=True,
1053
+ litellm_params=litellm_params,
1054
+ logger_fn=logger_fn,
1055
+ headers=prepped.headers,
1056
+ timeout=timeout,
1057
+ client=client,
1058
+ stream_chunk_size=stream_chunk_size,
1059
+ ) # type: ignore
1060
+ ### ASYNC COMPLETION
1061
+ return self.async_completion(
1062
+ model=model,
1063
+ messages=messages,
1064
+ data=data,
1065
+ api_base=proxy_endpoint_url,
1066
+ model_response=model_response,
1067
+ print_verbose=print_verbose,
1068
+ encoding=encoding,
1069
+ logging_obj=logging_obj,
1070
+ optional_params=optional_params,
1071
+ stream=stream, # type: ignore
1072
+ litellm_params=litellm_params,
1073
+ logger_fn=logger_fn,
1074
+ headers=prepped.headers,
1075
+ timeout=timeout,
1076
+ client=client,
1077
+ ) # type: ignore
1078
+
1079
+ if client is None or isinstance(client, AsyncHTTPHandler):
1080
+ _params = {}
1081
+ if timeout is not None:
1082
+ if isinstance(timeout, float) or isinstance(timeout, int):
1083
+ timeout = httpx.Timeout(timeout)
1084
+ _params["timeout"] = timeout
1085
+ self.client = _get_httpx_client(_params) # type: ignore
1086
+ else:
1087
+ self.client = client
1088
+ if (stream is not None and stream is True) and provider != "ai21":
1089
+ response = self.client.post(
1090
+ url=proxy_endpoint_url,
1091
+ headers=prepped.headers, # type: ignore
1092
+ data=data,
1093
+ stream=stream,
1094
+ logging_obj=logging_obj,
1095
+ )
1096
+
1097
+ if response.status_code != 200:
1098
+ raise BedrockError(
1099
+ status_code=response.status_code, message=str(response.read())
1100
+ )
1101
+
1102
+ decoder = AWSEventStreamDecoder(model=model)
1103
+
1104
+ completion_stream = decoder.iter_bytes(
1105
+ response.iter_bytes(chunk_size=stream_chunk_size)
1106
+ )
1107
+ streaming_response = CustomStreamWrapper(
1108
+ completion_stream=completion_stream,
1109
+ model=model,
1110
+ custom_llm_provider="bedrock",
1111
+ logging_obj=logging_obj,
1112
+ )
1113
+
1114
+ ## LOGGING
1115
+ logging_obj.post_call(
1116
+ input=messages,
1117
+ api_key="",
1118
+ original_response=streaming_response,
1119
+ additional_args={"complete_input_dict": data},
1120
+ )
1121
+ return streaming_response
1122
+
1123
+ try:
1124
+ response = self.client.post(
1125
+ url=proxy_endpoint_url,
1126
+ headers=dict(prepped.headers),
1127
+ data=data,
1128
+ logging_obj=logging_obj,
1129
+ )
1130
+ response.raise_for_status()
1131
+ except httpx.HTTPStatusError as err:
1132
+ error_code = err.response.status_code
1133
+ raise BedrockError(status_code=error_code, message=err.response.text)
1134
+ except httpx.TimeoutException:
1135
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
1136
+
1137
+ return self.process_response(
1138
+ model=model,
1139
+ response=response,
1140
+ model_response=model_response,
1141
+ stream=stream,
1142
+ logging_obj=logging_obj,
1143
+ optional_params=optional_params,
1144
+ api_key="",
1145
+ data=data,
1146
+ messages=messages,
1147
+ print_verbose=print_verbose,
1148
+ encoding=encoding,
1149
+ )
1150
+
1151
+ async def async_completion(
1152
+ self,
1153
+ model: str,
1154
+ messages: list,
1155
+ api_base: str,
1156
+ model_response: ModelResponse,
1157
+ print_verbose: Callable,
1158
+ data: str,
1159
+ timeout: Optional[Union[float, httpx.Timeout]],
1160
+ encoding,
1161
+ logging_obj: Logging,
1162
+ stream,
1163
+ optional_params: dict,
1164
+ litellm_params=None,
1165
+ logger_fn=None,
1166
+ headers={},
1167
+ client: Optional[AsyncHTTPHandler] = None,
1168
+ ) -> Union[ModelResponse, CustomStreamWrapper]:
1169
+ if client is None:
1170
+ _params = {}
1171
+ if timeout is not None:
1172
+ if isinstance(timeout, float) or isinstance(timeout, int):
1173
+ timeout = httpx.Timeout(timeout)
1174
+ _params["timeout"] = timeout
1175
+ client = get_async_httpx_client(params=_params, llm_provider=litellm.LlmProviders.BEDROCK) # type: ignore
1176
+ else:
1177
+ client = client # type: ignore
1178
+
1179
+ try:
1180
+ response = await client.post(
1181
+ api_base,
1182
+ headers=headers,
1183
+ data=data,
1184
+ timeout=timeout,
1185
+ logging_obj=logging_obj,
1186
+ )
1187
+ response.raise_for_status()
1188
+ except httpx.HTTPStatusError as err:
1189
+ error_code = err.response.status_code
1190
+ raise BedrockError(status_code=error_code, message=err.response.text)
1191
+ except httpx.TimeoutException:
1192
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
1193
+
1194
+ return self.process_response(
1195
+ model=model,
1196
+ response=response,
1197
+ model_response=model_response,
1198
+ stream=stream if isinstance(stream, bool) else False,
1199
+ logging_obj=logging_obj,
1200
+ api_key="",
1201
+ data=data,
1202
+ messages=messages,
1203
+ print_verbose=print_verbose,
1204
+ optional_params=optional_params,
1205
+ encoding=encoding,
1206
+ )
1207
+
1208
+ @track_llm_api_timing() # for streaming, we need to instrument the function calling the wrapper
1209
+ async def async_streaming(
1210
+ self,
1211
+ model: str,
1212
+ messages: list,
1213
+ api_base: str,
1214
+ model_response: ModelResponse,
1215
+ print_verbose: Callable,
1216
+ data: str,
1217
+ timeout: Optional[Union[float, httpx.Timeout]],
1218
+ encoding,
1219
+ logging_obj: Logging,
1220
+ stream,
1221
+ optional_params: dict,
1222
+ litellm_params=None,
1223
+ logger_fn=None,
1224
+ headers={},
1225
+ client: Optional[AsyncHTTPHandler] = None,
1226
+ stream_chunk_size: int = 1024,
1227
+ ) -> CustomStreamWrapper:
1228
+ # The call is not made here; instead, we prepare the necessary objects for the stream.
1229
+
1230
+ streaming_response = CustomStreamWrapper(
1231
+ completion_stream=None,
1232
+ make_call=partial(
1233
+ make_call,
1234
+ client=client,
1235
+ api_base=api_base,
1236
+ headers=headers,
1237
+ data=data, # type: ignore
1238
+ model=model,
1239
+ messages=messages,
1240
+ logging_obj=logging_obj,
1241
+ fake_stream=True if "ai21" in api_base else False,
1242
+ stream_chunk_size=stream_chunk_size,
1243
+ ),
1244
+ model=model,
1245
+ custom_llm_provider="bedrock",
1246
+ logging_obj=logging_obj,
1247
+ )
1248
+ return streaming_response
1249
+
1250
+ @staticmethod
1251
+ def _get_provider_from_model_path(
1252
+ model_path: str,
1253
+ ) -> Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL]:
1254
+ """
1255
+ Helper function to get the provider from a model path with format: provider/model-name
1256
+
1257
+ Args:
1258
+ model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
1259
+
1260
+ Returns:
1261
+ Optional[str]: The provider name, or None if no valid provider found
1262
+ """
1263
+ parts = model_path.split("/")
1264
+ if len(parts) >= 1:
1265
+ provider = parts[0]
1266
+ if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
1267
+ return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
1268
+ return None
1269
+
1270
+
1271
+ def get_response_stream_shape():
1272
+ global _response_stream_shape_cache
1273
+ if _response_stream_shape_cache is None:
1274
+ from botocore.loaders import Loader
1275
+ from botocore.model import ServiceModel
1276
+
1277
+ loader = Loader()
1278
+ bedrock_service_dict = loader.load_service_model("bedrock-runtime", "service-2")
1279
+ bedrock_service_model = ServiceModel(bedrock_service_dict)
1280
+ _response_stream_shape_cache = bedrock_service_model.shape_for("ResponseStream")
1281
+
1282
+ return _response_stream_shape_cache
1283
+
1284
+
1285
+ class AWSEventStreamDecoder:
1286
+ def __init__(self, model: str, json_mode: Optional[bool] = False) -> None:
1287
+ from botocore.parsers import EventStreamJSONParser
1288
+
1289
+ self.model = model
1290
+ self.parser = EventStreamJSONParser()
1291
+ self.content_blocks: List[ContentBlockDeltaEvent] = []
1292
+ self.tool_calls_index: Optional[int] = None
1293
+ self.response_id: Optional[str] = None
1294
+ self.json_mode = json_mode
1295
+ self._current_tool_name: Optional[str] = None
1296
+
1297
+ def check_empty_tool_call_args(self) -> bool:
1298
+ """
1299
+ Check if the tool call block so far has been an empty string
1300
+ """
1301
+ args = ""
1302
+ # if text content block -> skip
1303
+ if len(self.content_blocks) == 0:
1304
+ return False
1305
+
1306
+ if (
1307
+ "toolUse" not in self.content_blocks[0]
1308
+ ): # be explicit - only do this if tool use block, as this is to prevent json decoding errors
1309
+ return False
1310
+
1311
+ for block in self.content_blocks:
1312
+ if "toolUse" in block:
1313
+ args += block["toolUse"]["input"]
1314
+
1315
+ if len(args) == 0:
1316
+ return True
1317
+ return False
1318
+
1319
+ def extract_reasoning_content_str(
1320
+ self, reasoning_content_block: BedrockConverseReasoningContentBlockDelta
1321
+ ) -> Optional[str]:
1322
+ if "text" in reasoning_content_block:
1323
+ return reasoning_content_block["text"]
1324
+ return None
1325
+
1326
+ def translate_thinking_blocks(
1327
+ self, thinking_block: BedrockConverseReasoningContentBlockDelta
1328
+ ) -> Optional[
1329
+ List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]
1330
+ ]:
1331
+ """
1332
+ Translate the thinking blocks to a string
1333
+ """
1334
+
1335
+ thinking_blocks_list: List[
1336
+ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
1337
+ ] = []
1338
+ _thinking_block: Optional[
1339
+ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
1340
+ ] = None
1341
+
1342
+ if "text" in thinking_block:
1343
+ _thinking_block = ChatCompletionThinkingBlock(type="thinking")
1344
+ _thinking_block["thinking"] = thinking_block["text"]
1345
+ elif "signature" in thinking_block:
1346
+ _thinking_block = ChatCompletionThinkingBlock(type="thinking")
1347
+ _thinking_block["signature"] = thinking_block["signature"]
1348
+ _thinking_block["thinking"] = "" # consistent with anthropic response
1349
+ elif "redactedContent" in thinking_block:
1350
+ _thinking_block = ChatCompletionRedactedThinkingBlock(
1351
+ type="redacted_thinking", data=thinking_block["redactedContent"]
1352
+ )
1353
+ if _thinking_block is not None:
1354
+ thinking_blocks_list.append(_thinking_block)
1355
+ return thinking_blocks_list
1356
+
1357
+ def _initialize_converse_response_id(self, chunk_data: dict):
1358
+ """Initialize response_id from chunk data if not already set."""
1359
+ if self.response_id is None:
1360
+ if "messageStart" in chunk_data:
1361
+ conversation_id = chunk_data["messageStart"].get("conversationId")
1362
+ if conversation_id:
1363
+ self.response_id = f"chatcmpl-{conversation_id}"
1364
+ else:
1365
+ # Fallback to generating a UUID if the first chunk is not messageStart
1366
+ self.response_id = f"chatcmpl-{uuid.uuid4()}"
1367
+
1368
+ def _handle_converse_start_event(
1369
+ self,
1370
+ start_obj: ContentBlockStartEvent,
1371
+ ) -> Tuple[
1372
+ Optional[ChatCompletionToolCallChunk],
1373
+ dict,
1374
+ Optional[
1375
+ List[
1376
+ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
1377
+ ]
1378
+ ],
1379
+ ]:
1380
+ """Handle 'start' event in converse chunk parsing."""
1381
+ tool_use: Optional[ChatCompletionToolCallChunk] = None
1382
+ provider_specific_fields: dict = {}
1383
+ thinking_blocks: Optional[
1384
+ List[
1385
+ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
1386
+ ]
1387
+ ] = None
1388
+
1389
+ self.content_blocks = [] # reset
1390
+ if start_obj is not None:
1391
+ if "toolUse" in start_obj and start_obj["toolUse"] is not None:
1392
+ ## check tool name was formatted by litellm
1393
+ _response_tool_name = start_obj["toolUse"]["name"]
1394
+ response_tool_name = get_bedrock_tool_name(
1395
+ response_tool_name=_response_tool_name
1396
+ )
1397
+ self._current_tool_name = response_tool_name
1398
+
1399
+ # When json_mode is True, suppress the internal json_tool_call
1400
+ # and convert its content to text in delta events instead
1401
+ if (
1402
+ self.json_mode is True
1403
+ and response_tool_name == RESPONSE_FORMAT_TOOL_NAME
1404
+ ):
1405
+ return tool_use, provider_specific_fields, thinking_blocks
1406
+
1407
+ self.tool_calls_index = (
1408
+ 0 if self.tool_calls_index is None else self.tool_calls_index + 1
1409
+ )
1410
+ tool_use = {
1411
+ "id": start_obj["toolUse"]["toolUseId"],
1412
+ "type": "function",
1413
+ "function": {
1414
+ "name": response_tool_name,
1415
+ "arguments": "",
1416
+ },
1417
+ "index": self.tool_calls_index,
1418
+ }
1419
+ elif (
1420
+ "reasoningContent" in start_obj
1421
+ and start_obj["reasoningContent"] is not None
1422
+ ): # redacted thinking can be in start object
1423
+ thinking_blocks = self.translate_thinking_blocks(
1424
+ start_obj["reasoningContent"]
1425
+ )
1426
+ provider_specific_fields = {
1427
+ "reasoningContent": start_obj["reasoningContent"],
1428
+ }
1429
+ return tool_use, provider_specific_fields, thinking_blocks
1430
+
1431
+ def _handle_converse_delta_event(
1432
+ self,
1433
+ delta_obj: ContentBlockDeltaEvent,
1434
+ index: int,
1435
+ ) -> Tuple[
1436
+ str,
1437
+ Optional[ChatCompletionToolCallChunk],
1438
+ dict,
1439
+ Optional[str],
1440
+ Optional[
1441
+ List[
1442
+ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
1443
+ ]
1444
+ ],
1445
+ ]:
1446
+ """Handle 'delta' event in converse chunk parsing."""
1447
+ text = ""
1448
+ tool_use: Optional[ChatCompletionToolCallChunk] = None
1449
+ provider_specific_fields: dict = {}
1450
+ reasoning_content: Optional[str] = None
1451
+ thinking_blocks: Optional[
1452
+ List[
1453
+ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
1454
+ ]
1455
+ ] = None
1456
+
1457
+ self.content_blocks.append(delta_obj)
1458
+ if "text" in delta_obj:
1459
+ text = delta_obj["text"]
1460
+ elif "toolUse" in delta_obj:
1461
+ # When json_mode is True and this is the internal json_tool_call,
1462
+ # convert tool input to text content instead of tool call arguments
1463
+ if (
1464
+ self.json_mode is True
1465
+ and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME
1466
+ ):
1467
+ text = delta_obj["toolUse"]["input"]
1468
+ else:
1469
+ tool_use = {
1470
+ "id": None,
1471
+ "type": "function",
1472
+ "function": {
1473
+ "name": None,
1474
+ "arguments": delta_obj["toolUse"]["input"],
1475
+ },
1476
+ "index": (
1477
+ self.tool_calls_index
1478
+ if self.tool_calls_index is not None
1479
+ else index
1480
+ ),
1481
+ }
1482
+ elif "reasoningContent" in delta_obj:
1483
+ provider_specific_fields = {
1484
+ "reasoningContent": delta_obj["reasoningContent"],
1485
+ }
1486
+ reasoning_content = self.extract_reasoning_content_str(
1487
+ delta_obj["reasoningContent"]
1488
+ )
1489
+ thinking_blocks = self.translate_thinking_blocks(
1490
+ delta_obj["reasoningContent"]
1491
+ )
1492
+ if (
1493
+ thinking_blocks
1494
+ and len(thinking_blocks) > 0
1495
+ and reasoning_content is None
1496
+ ):
1497
+ reasoning_content = (
1498
+ "" # set to non-empty string to ensure consistency with Anthropic
1499
+ )
1500
+ elif "citationsContent" in delta_obj:
1501
+ # Handle Nova grounding citations in streaming responses
1502
+ provider_specific_fields = {
1503
+ "citationsContent": delta_obj["citationsContent"],
1504
+ }
1505
+ return (
1506
+ text,
1507
+ tool_use,
1508
+ provider_specific_fields,
1509
+ reasoning_content,
1510
+ thinking_blocks,
1511
+ )
1512
+
1513
+ def _handle_converse_stop_event(
1514
+ self, index: int
1515
+ ) -> Optional[ChatCompletionToolCallChunk]:
1516
+ """Handle stop/contentBlockIndex event in converse chunk parsing."""
1517
+ tool_use: Optional[ChatCompletionToolCallChunk] = None
1518
+
1519
+ # If the ending block was the internal json_tool_call, skip emitting
1520
+ # the empty-args tool chunk and reset tracking state
1521
+ if (
1522
+ self.json_mode is True
1523
+ and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME
1524
+ ):
1525
+ self._current_tool_name = None
1526
+ return tool_use
1527
+
1528
+ self._current_tool_name = None
1529
+ is_empty = self.check_empty_tool_call_args()
1530
+ if is_empty:
1531
+ tool_use = {
1532
+ "id": None,
1533
+ "type": "function",
1534
+ "function": {
1535
+ "name": None,
1536
+ "arguments": "{}",
1537
+ },
1538
+ "index": (
1539
+ self.tool_calls_index
1540
+ if self.tool_calls_index is not None
1541
+ else index
1542
+ ),
1543
+ }
1544
+ return tool_use
1545
+
1546
+ def converse_chunk_parser(self, chunk_data: dict) -> ModelResponseStream:
1547
+ try:
1548
+ # Capture the conversationId from the first messageStart event
1549
+ # and use it as the consistent ID for all subsequent chunks.
1550
+ self._initialize_converse_response_id(chunk_data)
1551
+
1552
+ verbose_logger.debug("\n\nRaw Chunk: {}\n\n".format(chunk_data))
1553
+ text = ""
1554
+ tool_use: Optional[ChatCompletionToolCallChunk] = None
1555
+ finish_reason = ""
1556
+ usage: Optional[Usage] = None
1557
+ provider_specific_fields: dict = {}
1558
+ reasoning_content: Optional[str] = None
1559
+ thinking_blocks: Optional[
1560
+ List[
1561
+ Union[
1562
+ ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock
1563
+ ]
1564
+ ]
1565
+ ] = None
1566
+
1567
+ content_block_index = int(chunk_data.get("contentBlockIndex", 0))
1568
+ if "start" in chunk_data:
1569
+ start_obj = ContentBlockStartEvent(**chunk_data["start"])
1570
+ (
1571
+ tool_use,
1572
+ provider_specific_fields,
1573
+ thinking_blocks,
1574
+ ) = self._handle_converse_start_event(start_obj)
1575
+ elif "delta" in chunk_data:
1576
+ delta_obj = ContentBlockDeltaEvent(**chunk_data["delta"])
1577
+ (
1578
+ text,
1579
+ tool_use,
1580
+ provider_specific_fields,
1581
+ reasoning_content,
1582
+ thinking_blocks,
1583
+ ) = self._handle_converse_delta_event(delta_obj, content_block_index)
1584
+ elif (
1585
+ "contentBlockIndex" in chunk_data
1586
+ ): # stop block, no 'start' or 'delta' object
1587
+ tool_use = self._handle_converse_stop_event(content_block_index)
1588
+ elif "stopReason" in chunk_data:
1589
+ finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
1590
+ elif "usage" in chunk_data:
1591
+ usage = converse_config._transform_usage(chunk_data.get("usage", {}))
1592
+
1593
+ model_response_provider_specific_fields = {}
1594
+ if "trace" in chunk_data:
1595
+ trace = chunk_data.get("trace")
1596
+ model_response_provider_specific_fields["trace"] = trace
1597
+ response = ModelResponseStream(
1598
+ choices=[
1599
+ StreamingChoices(
1600
+ finish_reason=finish_reason,
1601
+ index=0, # Always 0 - Bedrock never returns multiple choices
1602
+ delta=Delta(
1603
+ content=text,
1604
+ role="assistant",
1605
+ tool_calls=[tool_use] if tool_use else None,
1606
+ provider_specific_fields=(
1607
+ provider_specific_fields
1608
+ if provider_specific_fields
1609
+ else None
1610
+ ),
1611
+ thinking_blocks=thinking_blocks,
1612
+ reasoning_content=reasoning_content,
1613
+ ),
1614
+ )
1615
+ ],
1616
+ id=self.response_id,
1617
+ model=self.model,
1618
+ usage=usage,
1619
+ provider_specific_fields=model_response_provider_specific_fields,
1620
+ )
1621
+
1622
+ return response
1623
+ except Exception as e:
1624
+ raise Exception("Received streaming error - {}".format(str(e)))
1625
+
1626
+ def _chunk_parser(
1627
+ self, chunk_data: dict
1628
+ ) -> Union[GChunk, ModelResponseStream, dict]:
1629
+ text = ""
1630
+ is_finished = False
1631
+ finish_reason = ""
1632
+ if "outputText" in chunk_data:
1633
+ text = chunk_data["outputText"]
1634
+ # ai21 mapping
1635
+ elif "ai21" in self.model: # fake ai21 streaming
1636
+ text = chunk_data.get("completions")[0].get("data").get("text") # type: ignore
1637
+ is_finished = True
1638
+ finish_reason = "stop"
1639
+ ######## /bedrock/converse mappings ###############
1640
+ elif (
1641
+ "contentBlockIndex" in chunk_data
1642
+ or "stopReason" in chunk_data
1643
+ or "metrics" in chunk_data
1644
+ or "trace" in chunk_data
1645
+ ):
1646
+ return self.converse_chunk_parser(chunk_data=chunk_data)
1647
+ ######### /bedrock/invoke nova mappings ###############
1648
+ elif "contentBlockDelta" in chunk_data:
1649
+ # when using /bedrock/invoke/nova, the chunk_data is nested under "contentBlockDelta"
1650
+ _chunk_data = chunk_data.get("contentBlockDelta", {})
1651
+ return self.converse_chunk_parser(chunk_data=_chunk_data)
1652
+ ######## bedrock.mistral mappings ###############
1653
+ elif "outputs" in chunk_data:
1654
+ if (
1655
+ len(chunk_data["outputs"]) == 1
1656
+ and chunk_data["outputs"][0].get("text", None) is not None
1657
+ ):
1658
+ text = chunk_data["outputs"][0]["text"]
1659
+ stop_reason = chunk_data.get("stop_reason", None)
1660
+ if stop_reason is not None:
1661
+ is_finished = True
1662
+ finish_reason = stop_reason
1663
+ ######## bedrock.cohere mappings ###############
1664
+ # meta mapping
1665
+ elif "generation" in chunk_data:
1666
+ text = chunk_data["generation"] # bedrock.meta
1667
+ # cohere mapping
1668
+ elif "text" in chunk_data:
1669
+ text = chunk_data["text"] # bedrock.cohere
1670
+ # cohere mapping for finish reason
1671
+ elif "finish_reason" in chunk_data:
1672
+ finish_reason = chunk_data["finish_reason"]
1673
+ is_finished = True
1674
+ elif chunk_data.get("completionReason", None):
1675
+ is_finished = True
1676
+ finish_reason = chunk_data["completionReason"]
1677
+ return GChunk(
1678
+ text=text,
1679
+ is_finished=is_finished,
1680
+ finish_reason=finish_reason,
1681
+ usage=None,
1682
+ index=0,
1683
+ tool_use=None,
1684
+ )
1685
+
1686
+ def iter_bytes(
1687
+ self, iterator: Iterator[bytes]
1688
+ ) -> Iterator[Union[GChunk, ModelResponseStream, dict]]:
1689
+ """Given an iterator that yields lines, iterate over it & yield every event encountered"""
1690
+ from botocore.eventstream import EventStreamBuffer
1691
+
1692
+ event_stream_buffer = EventStreamBuffer()
1693
+ for chunk in iterator:
1694
+ event_stream_buffer.add_data(chunk)
1695
+ for event in event_stream_buffer:
1696
+ message = self._parse_message_from_event(event)
1697
+ if message:
1698
+ # sse_event = ServerSentEvent(data=message, event="completion")
1699
+ _data = json.loads(message)
1700
+ yield self._chunk_parser(chunk_data=_data)
1701
+
1702
+ async def aiter_bytes(
1703
+ self, iterator: AsyncIterator[bytes]
1704
+ ) -> AsyncIterator[Union[GChunk, ModelResponseStream, dict]]:
1705
+ """Given an async iterator that yields lines, iterate over it & yield every event encountered"""
1706
+ from botocore.eventstream import EventStreamBuffer
1707
+
1708
+ event_stream_buffer = EventStreamBuffer()
1709
+ async for chunk in iterator:
1710
+ event_stream_buffer.add_data(chunk)
1711
+ for event in event_stream_buffer:
1712
+ message = self._parse_message_from_event(event)
1713
+ if message:
1714
+ _data = json.loads(message)
1715
+ yield self._chunk_parser(chunk_data=_data)
1716
+
1717
+ def _parse_message_from_event(self, event) -> Optional[str]:
1718
+ response_dict = event.to_response_dict()
1719
+ parsed_response = self.parser.parse(response_dict, get_response_stream_shape())
1720
+
1721
+ if response_dict["status_code"] != 200:
1722
+ decoded_body = response_dict["body"].decode()
1723
+ if isinstance(decoded_body, dict):
1724
+ error_message = decoded_body.get("message")
1725
+ elif isinstance(decoded_body, str):
1726
+ error_message = decoded_body
1727
+ else:
1728
+ error_message = ""
1729
+ exception_status = response_dict["headers"].get(":exception-type")
1730
+ error_message = exception_status + " " + error_message
1731
+ raise BedrockError(
1732
+ status_code=response_dict["status_code"],
1733
+ message=(
1734
+ json.dumps(error_message)
1735
+ if isinstance(error_message, dict)
1736
+ else error_message
1737
+ ),
1738
+ )
1739
+ if "chunk" in parsed_response:
1740
+ chunk = parsed_response.get("chunk")
1741
+ if not chunk:
1742
+ return None
1743
+ return chunk.get("bytes").decode() # type: ignore[no-any-return]
1744
+ else:
1745
+ chunk = response_dict.get("body")
1746
+ if not chunk:
1747
+ return None
1748
+
1749
+ return chunk.decode() # type: ignore[no-any-return]
1750
+
1751
+
1752
+ class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder):
1753
+ def __init__(
1754
+ self,
1755
+ model: str,
1756
+ sync_stream: bool,
1757
+ json_mode: Optional[bool] = None,
1758
+ ) -> None:
1759
+ """
1760
+ Child class of AWSEventStreamDecoder that handles the streaming response from the Anthropic family of models
1761
+
1762
+ The only difference between AWSEventStreamDecoder and AmazonAnthropicClaudeStreamDecoder is the `chunk_parser` method
1763
+ """
1764
+ super().__init__(model=model)
1765
+ self.anthropic_model_response_iterator = AnthropicModelResponseIterator(
1766
+ streaming_response=None,
1767
+ sync_stream=sync_stream,
1768
+ json_mode=json_mode,
1769
+ )
1770
+
1771
+ def _chunk_parser(self, chunk_data: dict) -> ModelResponseStream:
1772
+ return self.anthropic_model_response_iterator.chunk_parser(chunk=chunk_data)
1773
+
1774
+
1775
+ class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder):
1776
+ def __init__(
1777
+ self,
1778
+ model: str,
1779
+ sync_stream: bool,
1780
+ ) -> None:
1781
+ super().__init__(model=model)
1782
+ from litellm.llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import (
1783
+ AmazonDeepseekR1ResponseIterator,
1784
+ )
1785
+
1786
+ self.deepseek_model_response_iterator = AmazonDeepseekR1ResponseIterator(
1787
+ streaming_response=None,
1788
+ sync_stream=sync_stream,
1789
+ )
1790
+
1791
+ def _chunk_parser(
1792
+ self, chunk_data: dict
1793
+ ) -> Union[GChunk, ModelResponseStream, dict]:
1794
+ return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)
1795
+
1796
+
1797
+ class MockResponseIterator: # for returning ai21 streaming responses
1798
+ def __init__(self, model_response, json_mode: Optional[bool] = False):
1799
+ self.model_response = model_response
1800
+ self.json_mode = json_mode
1801
+ self.is_done = False
1802
+
1803
+ # Sync iterator
1804
+ def __iter__(self):
1805
+ return self
1806
+
1807
+ def _handle_json_mode_chunk(
1808
+ self, text: str, tool_calls: Optional[List[ChatCompletionToolCallChunk]]
1809
+ ) -> Tuple[str, Optional[ChatCompletionToolCallChunk]]:
1810
+ """
1811
+ If JSON mode is enabled, convert the tool call to a message.
1812
+
1813
+ Bedrock returns the JSON schema as part of the tool call
1814
+ OpenAI returns the JSON schema as part of the content, this handles placing it in the content
1815
+
1816
+ Args:
1817
+ text: str
1818
+ tool_use: Optional[ChatCompletionToolCallChunk]
1819
+ Returns:
1820
+ Tuple[str, Optional[ChatCompletionToolCallChunk]]
1821
+
1822
+ text: The text to use in the content
1823
+ tool_use: The ChatCompletionToolCallChunk to use in the chunk response
1824
+ """
1825
+ tool_use: Optional[ChatCompletionToolCallChunk] = None
1826
+ if self.json_mode is True and tool_calls is not None:
1827
+ message = litellm.AnthropicConfig()._convert_tool_response_to_message(
1828
+ tool_calls=tool_calls
1829
+ )
1830
+ if message is not None:
1831
+ text = message.content or ""
1832
+ tool_use = None
1833
+ elif tool_calls is not None and len(tool_calls) > 0:
1834
+ tool_use = tool_calls[0]
1835
+ return text, tool_use
1836
+
1837
+ def _chunk_parser(self, chunk_data: ModelResponse) -> GChunk:
1838
+ try:
1839
+ chunk_usage: Usage = getattr(chunk_data, "usage")
1840
+ text = chunk_data.choices[0].message.content or "" # type: ignore
1841
+ tool_use = None
1842
+ _model_response_tool_call = cast(
1843
+ Optional[List[ChatCompletionMessageToolCall]],
1844
+ cast(Choices, chunk_data.choices[0]).message.tool_calls,
1845
+ )
1846
+ if self.json_mode is True:
1847
+ text, tool_use = self._handle_json_mode_chunk(
1848
+ text=text,
1849
+ tool_calls=chunk_data.choices[0].message.tool_calls, # type: ignore
1850
+ )
1851
+ elif _model_response_tool_call is not None:
1852
+ tool_use = ChatCompletionToolCallChunk(
1853
+ id=_model_response_tool_call[0].id,
1854
+ type="function",
1855
+ function=ChatCompletionToolCallFunctionChunk(
1856
+ name=_model_response_tool_call[0].function.name,
1857
+ arguments=_model_response_tool_call[0].function.arguments,
1858
+ ),
1859
+ index=0,
1860
+ )
1861
+ processed_chunk = GChunk(
1862
+ text=text,
1863
+ tool_use=tool_use,
1864
+ is_finished=True,
1865
+ finish_reason=map_finish_reason(
1866
+ finish_reason=chunk_data.choices[0].finish_reason or ""
1867
+ ),
1868
+ usage=ChatCompletionUsageBlock(
1869
+ prompt_tokens=chunk_usage.prompt_tokens,
1870
+ completion_tokens=chunk_usage.completion_tokens,
1871
+ total_tokens=chunk_usage.total_tokens,
1872
+ ),
1873
+ index=0,
1874
+ )
1875
+ return processed_chunk
1876
+ except Exception as e:
1877
+ raise ValueError(f"Failed to decode chunk: {chunk_data}. Error: {e}")
1878
+
1879
+ def __next__(self):
1880
+ if self.is_done:
1881
+ raise StopIteration
1882
+ self.is_done = True
1883
+ return self._chunk_parser(self.model_response)
1884
+
1885
+ # Async iterator
1886
+ def __aiter__(self):
1887
+ return self
1888
+
1889
+ async def __anext__(self):
1890
+ if self.is_done:
1891
+ raise StopAsyncIteration
1892
+ self.is_done = True
1893
+ return self._chunk_parser(self.model_response)
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import types
2
+ from typing import List, Optional
3
+
4
+ from litellm.llms.base_llm.chat.transformation import BaseConfig
5
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
6
+ AmazonInvokeConfig,
7
+ )
8
+
9
+
10
+ class AmazonAI21Config(AmazonInvokeConfig, BaseConfig):
11
+ """
12
+ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=j2-ultra
13
+
14
+ Supported Params for the Amazon / AI21 models:
15
+
16
+ - `maxTokens` (int32): The maximum number of tokens to generate per result. Optional, default is 16. If no `stopSequences` are given, generation stops after producing `maxTokens`.
17
+
18
+ - `temperature` (float): Modifies the distribution from which tokens are sampled. Optional, default is 0.7. A value of 0 essentially disables sampling and results in greedy decoding.
19
+
20
+ - `topP` (float): Used for sampling tokens from the corresponding top percentile of probability mass. Optional, default is 1. For instance, a value of 0.9 considers only tokens comprising the top 90% probability mass.
21
+
22
+ - `stopSequences` (array of strings): Stops decoding if any of the input strings is generated. Optional.
23
+
24
+ - `frequencyPenalty` (object): Placeholder for frequency penalty object.
25
+
26
+ - `presencePenalty` (object): Placeholder for presence penalty object.
27
+
28
+ - `countPenalty` (object): Placeholder for count penalty object.
29
+ """
30
+
31
+ maxTokens: Optional[int] = None
32
+ temperature: Optional[float] = None
33
+ topP: Optional[float] = None
34
+ stopSequences: Optional[list] = None
35
+ frequencePenalty: Optional[dict] = None
36
+ presencePenalty: Optional[dict] = None
37
+ countPenalty: Optional[dict] = None
38
+
39
+ def __init__(
40
+ self,
41
+ maxTokens: Optional[int] = None,
42
+ temperature: Optional[float] = None,
43
+ topP: Optional[float] = None,
44
+ stopSequences: Optional[list] = None,
45
+ frequencePenalty: Optional[dict] = None,
46
+ presencePenalty: Optional[dict] = None,
47
+ countPenalty: Optional[dict] = None,
48
+ ) -> None:
49
+ locals_ = locals().copy()
50
+ for key, value in locals_.items():
51
+ if key != "self" and value is not None:
52
+ setattr(self.__class__, key, value)
53
+
54
+ AmazonInvokeConfig.__init__(self)
55
+
56
+ @classmethod
57
+ def get_config(cls):
58
+ return {
59
+ k: v
60
+ for k, v in cls.__dict__.items()
61
+ if not k.startswith("__")
62
+ and not k.startswith("_abc")
63
+ and not isinstance(
64
+ v,
65
+ (
66
+ types.FunctionType,
67
+ types.BuiltinFunctionType,
68
+ classmethod,
69
+ staticmethod,
70
+ ),
71
+ )
72
+ and v is not None
73
+ }
74
+
75
+ def get_supported_openai_params(self, model: str) -> List:
76
+ return [
77
+ "max_tokens",
78
+ "temperature",
79
+ "top_p",
80
+ "stream",
81
+ ]
82
+
83
+ def map_openai_params(
84
+ self,
85
+ non_default_params: dict,
86
+ optional_params: dict,
87
+ model: str,
88
+ drop_params: bool,
89
+ ) -> dict:
90
+ for k, v in non_default_params.items():
91
+ if k == "max_tokens":
92
+ optional_params["maxTokens"] = v
93
+ if k == "temperature":
94
+ optional_params["temperature"] = v
95
+ if k == "top_p":
96
+ optional_params["topP"] = v
97
+ if k == "stream":
98
+ optional_params["stream"] = v
99
+ return optional_params
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import types
2
+ from typing import List, Optional
3
+
4
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
5
+ AmazonInvokeConfig,
6
+ )
7
+ from litellm.llms.cohere.chat.transformation import CohereChatConfig
8
+
9
+
10
+ class AmazonCohereConfig(AmazonInvokeConfig, CohereChatConfig):
11
+ """
12
+ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=command
13
+
14
+ Supported Params for the Amazon / Cohere models:
15
+
16
+ - `max_tokens` (integer) max tokens,
17
+ - `temperature` (float) model temperature,
18
+ - `return_likelihood` (string) n/a
19
+ """
20
+
21
+ max_tokens: Optional[int] = None
22
+ return_likelihood: Optional[str] = None
23
+
24
+ def __init__(
25
+ self,
26
+ max_tokens: Optional[int] = None,
27
+ temperature: Optional[float] = None,
28
+ return_likelihood: Optional[str] = None,
29
+ ) -> None:
30
+ locals_ = locals().copy()
31
+ for key, value in locals_.items():
32
+ if key != "self" and value is not None:
33
+ setattr(self.__class__, key, value)
34
+
35
+ AmazonInvokeConfig.__init__(self)
36
+
37
+ @classmethod
38
+ def get_config(cls):
39
+ return {
40
+ k: v
41
+ for k, v in cls.__dict__.items()
42
+ if not k.startswith("__")
43
+ and not k.startswith("_abc")
44
+ and not isinstance(
45
+ v,
46
+ (
47
+ types.FunctionType,
48
+ types.BuiltinFunctionType,
49
+ classmethod,
50
+ staticmethod,
51
+ ),
52
+ )
53
+ and v is not None
54
+ }
55
+
56
+ def get_supported_openai_params(self, model: str) -> List[str]:
57
+ supported_params = CohereChatConfig.get_supported_openai_params(
58
+ self, model=model
59
+ )
60
+ return supported_params
61
+
62
+ def map_openai_params(
63
+ self,
64
+ non_default_params: dict,
65
+ optional_params: dict,
66
+ model: str,
67
+ drop_params: bool,
68
+ ) -> dict:
69
+ return CohereChatConfig.map_openai_params(
70
+ self,
71
+ non_default_params=non_default_params,
72
+ optional_params=optional_params,
73
+ model=model,
74
+ drop_params=drop_params,
75
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, List, Optional, cast
2
+
3
+ from httpx import Response
4
+
5
+ from litellm import verbose_logger
6
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
7
+ _parse_content_for_reasoning,
8
+ )
9
+ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
10
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
11
+ LiteLLMLoggingObj,
12
+ )
13
+ from litellm.types.llms.bedrock import AmazonDeepSeekR1StreamingResponse
14
+ from litellm.types.llms.openai import AllMessageValues
15
+ from litellm.types.utils import (
16
+ ChatCompletionUsageBlock,
17
+ Choices,
18
+ Delta,
19
+ Message,
20
+ ModelResponse,
21
+ ModelResponseStream,
22
+ StreamingChoices,
23
+ )
24
+
25
+ from .amazon_llama_transformation import AmazonLlamaConfig
26
+
27
+
28
+ class AmazonDeepSeekR1Config(AmazonLlamaConfig):
29
+ def transform_response(
30
+ self,
31
+ model: str,
32
+ raw_response: Response,
33
+ model_response: ModelResponse,
34
+ logging_obj: LiteLLMLoggingObj,
35
+ request_data: dict,
36
+ messages: List[AllMessageValues],
37
+ optional_params: dict,
38
+ litellm_params: dict,
39
+ encoding: Any,
40
+ api_key: Optional[str] = None,
41
+ json_mode: Optional[bool] = None,
42
+ ) -> ModelResponse:
43
+ """
44
+ Extract the reasoning content, and return it as a separate field in the response.
45
+ """
46
+ response = super().transform_response(
47
+ model,
48
+ raw_response,
49
+ model_response,
50
+ logging_obj,
51
+ request_data,
52
+ messages,
53
+ optional_params,
54
+ litellm_params,
55
+ encoding,
56
+ api_key,
57
+ json_mode,
58
+ )
59
+ prompt = cast(Optional[str], request_data.get("prompt"))
60
+ message_content = cast(
61
+ Optional[str], cast(Choices, response.choices[0]).message.get("content")
62
+ )
63
+ if prompt and prompt.strip().endswith("<think>") and message_content:
64
+ message_content_with_reasoning_token = "<think>" + message_content
65
+ reasoning, content = _parse_content_for_reasoning(
66
+ message_content_with_reasoning_token
67
+ )
68
+ provider_specific_fields = (
69
+ cast(Choices, response.choices[0]).message.provider_specific_fields
70
+ or {}
71
+ )
72
+ if reasoning:
73
+ provider_specific_fields["reasoning_content"] = reasoning
74
+
75
+ message = Message(
76
+ **{
77
+ **cast(Choices, response.choices[0]).message.model_dump(),
78
+ "content": content,
79
+ "provider_specific_fields": provider_specific_fields,
80
+ }
81
+ )
82
+ cast(Choices, response.choices[0]).message = message
83
+ return response
84
+
85
+
86
+ class AmazonDeepseekR1ResponseIterator(BaseModelResponseIterator):
87
+ def __init__(self, streaming_response: Any, sync_stream: bool) -> None:
88
+ super().__init__(streaming_response=streaming_response, sync_stream=sync_stream)
89
+ self.has_finished_thinking = False
90
+
91
+ def chunk_parser(self, chunk: dict) -> ModelResponseStream:
92
+ """
93
+ Deepseek r1 starts by thinking, then it generates the response.
94
+ """
95
+ try:
96
+ typed_chunk = AmazonDeepSeekR1StreamingResponse(**chunk) # type: ignore
97
+ generated_content = typed_chunk["generation"]
98
+ if generated_content == "</think>" and not self.has_finished_thinking:
99
+ verbose_logger.debug(
100
+ "Deepseek r1: </think> received, setting has_finished_thinking to True"
101
+ )
102
+ generated_content = ""
103
+ self.has_finished_thinking = True
104
+
105
+ prompt_token_count = typed_chunk.get("prompt_token_count") or 0
106
+ generation_token_count = typed_chunk.get("generation_token_count") or 0
107
+ usage = ChatCompletionUsageBlock(
108
+ prompt_tokens=prompt_token_count,
109
+ completion_tokens=generation_token_count,
110
+ total_tokens=prompt_token_count + generation_token_count,
111
+ )
112
+
113
+ return ModelResponseStream(
114
+ choices=[
115
+ StreamingChoices(
116
+ finish_reason=typed_chunk["stop_reason"],
117
+ delta=Delta(
118
+ content=(
119
+ generated_content
120
+ if self.has_finished_thinking
121
+ else None
122
+ ),
123
+ reasoning_content=(
124
+ generated_content
125
+ if not self.has_finished_thinking
126
+ else None
127
+ ),
128
+ ),
129
+ )
130
+ ],
131
+ usage=usage,
132
+ )
133
+
134
+ except Exception as e:
135
+ raise e
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import types
2
+ from typing import List, Optional
3
+
4
+ from litellm.llms.base_llm.chat.transformation import BaseConfig
5
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
6
+ AmazonInvokeConfig,
7
+ )
8
+
9
+
10
+ class AmazonLlamaConfig(AmazonInvokeConfig, BaseConfig):
11
+ """
12
+ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=meta.llama2-13b-chat-v1
13
+
14
+ Supported Params for the Amazon / Meta Llama models:
15
+
16
+ - `max_gen_len` (integer) max tokens,
17
+ - `temperature` (float) temperature for model,
18
+ - `top_p` (float) top p for model
19
+ """
20
+
21
+ max_gen_len: Optional[int] = None
22
+ temperature: Optional[float] = None
23
+ topP: Optional[float] = None
24
+
25
+ def __init__(
26
+ self,
27
+ maxTokenCount: Optional[int] = None,
28
+ temperature: Optional[float] = None,
29
+ topP: Optional[int] = None,
30
+ ) -> None:
31
+ locals_ = locals().copy()
32
+ for key, value in locals_.items():
33
+ if key != "self" and value is not None:
34
+ setattr(self.__class__, key, value)
35
+ AmazonInvokeConfig.__init__(self)
36
+
37
+ @classmethod
38
+ def get_config(cls):
39
+ return {
40
+ k: v
41
+ for k, v in cls.__dict__.items()
42
+ if not k.startswith("__")
43
+ and not k.startswith("_abc")
44
+ and not isinstance(
45
+ v,
46
+ (
47
+ types.FunctionType,
48
+ types.BuiltinFunctionType,
49
+ classmethod,
50
+ staticmethod,
51
+ ),
52
+ )
53
+ and v is not None
54
+ }
55
+
56
+ def get_supported_openai_params(self, model: str) -> List:
57
+ return [
58
+ "max_tokens",
59
+ "temperature",
60
+ "top_p",
61
+ "stream",
62
+ ]
63
+
64
+ def map_openai_params(
65
+ self,
66
+ non_default_params: dict,
67
+ optional_params: dict,
68
+ model: str,
69
+ drop_params: bool,
70
+ ) -> dict:
71
+ for k, v in non_default_params.items():
72
+ if k == "max_tokens":
73
+ optional_params["max_gen_len"] = v
74
+ if k == "temperature":
75
+ optional_params["temperature"] = v
76
+ if k == "top_p":
77
+ optional_params["top_p"] = v
78
+ if k == "stream":
79
+ optional_params["stream"] = v
80
+ return optional_params
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import types
2
+ from typing import List, Optional, TYPE_CHECKING
3
+
4
+ from litellm.llms.base_llm.chat.transformation import BaseConfig
5
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
6
+ AmazonInvokeConfig,
7
+ )
8
+ from litellm.llms.bedrock.common_utils import BedrockError
9
+
10
+ if TYPE_CHECKING:
11
+ from litellm.types.utils import ModelResponse
12
+
13
+
14
+ class AmazonMistralConfig(AmazonInvokeConfig, BaseConfig):
15
+ """
16
+ Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mistral.html
17
+ Supported Params for the Amazon / Mistral models:
18
+
19
+ - `max_tokens` (integer) max tokens,
20
+ - `temperature` (float) temperature for model,
21
+ - `top_p` (float) top p for model
22
+ - `stop` [string] A list of stop sequences that if generated by the model, stops the model from generating further output.
23
+ - `top_k` (float) top k for model
24
+ """
25
+
26
+ max_tokens: Optional[int] = None
27
+ temperature: Optional[float] = None
28
+ top_p: Optional[float] = None
29
+ top_k: Optional[float] = None
30
+ stop: Optional[List[str]] = None
31
+
32
+ def __init__(
33
+ self,
34
+ max_tokens: Optional[int] = None,
35
+ temperature: Optional[float] = None,
36
+ top_p: Optional[int] = None,
37
+ top_k: Optional[float] = None,
38
+ stop: Optional[List[str]] = None,
39
+ ) -> None:
40
+ locals_ = locals().copy()
41
+ for key, value in locals_.items():
42
+ if key != "self" and value is not None:
43
+ setattr(self.__class__, key, value)
44
+
45
+ AmazonInvokeConfig.__init__(self)
46
+
47
+ @classmethod
48
+ def get_config(cls):
49
+ return {
50
+ k: v
51
+ for k, v in cls.__dict__.items()
52
+ if not k.startswith("__")
53
+ and not k.startswith("_abc")
54
+ and not isinstance(
55
+ v,
56
+ (
57
+ types.FunctionType,
58
+ types.BuiltinFunctionType,
59
+ classmethod,
60
+ staticmethod,
61
+ ),
62
+ )
63
+ and v is not None
64
+ }
65
+
66
+ def get_supported_openai_params(self, model: str) -> List[str]:
67
+ return ["max_tokens", "temperature", "top_p", "stop", "stream"]
68
+
69
+ def map_openai_params(
70
+ self,
71
+ non_default_params: dict,
72
+ optional_params: dict,
73
+ model: str,
74
+ drop_params: bool,
75
+ ) -> dict:
76
+ for k, v in non_default_params.items():
77
+ if k == "max_tokens":
78
+ optional_params["max_tokens"] = v
79
+ if k == "temperature":
80
+ optional_params["temperature"] = v
81
+ if k == "top_p":
82
+ optional_params["top_p"] = v
83
+ if k == "stop":
84
+ optional_params["stop"] = v
85
+ if k == "stream":
86
+ optional_params["stream"] = v
87
+ return optional_params
88
+
89
+ @staticmethod
90
+ def get_outputText(
91
+ completion_response: dict, model_response: "ModelResponse"
92
+ ) -> str:
93
+ """This function extracts the output text from a bedrock mistral completion.
94
+ As a side effect, it updates the finish reason for a model response.
95
+
96
+ Args:
97
+ completion_response: JSON from the completion.
98
+ model_response: ModelResponse
99
+
100
+ Returns:
101
+ A string with the response of the LLM
102
+
103
+ """
104
+ if "choices" in completion_response:
105
+ outputText = completion_response["choices"][0]["message"]["content"]
106
+ model_response.choices[0].finish_reason = completion_response["choices"][0][
107
+ "finish_reason"
108
+ ]
109
+ elif "outputs" in completion_response:
110
+ outputText = completion_response["outputs"][0]["text"]
111
+ model_response.choices[0].finish_reason = completion_response["outputs"][0][
112
+ "stop_reason"
113
+ ]
114
+ else:
115
+ raise BedrockError(
116
+ message="Unexpected mistral completion response", status_code=400
117
+ )
118
+
119
+ return outputText
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py ADDED
@@ -0,0 +1,266 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation for Bedrock Moonshot AI (Kimi K2) models.
3
+
4
+ Supports the Kimi K2 Thinking model available on Amazon Bedrock.
5
+ Model format: bedrock/moonshot.kimi-k2-thinking-v1:0
6
+
7
+ Reference: https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/
8
+ """
9
+
10
+ from typing import TYPE_CHECKING, Any, List, Optional, Union
11
+ import re
12
+
13
+ import httpx
14
+
15
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
16
+ AmazonInvokeConfig,
17
+ )
18
+ from litellm.llms.bedrock.common_utils import BedrockError
19
+ from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig
20
+ from litellm.types.llms.openai import AllMessageValues
21
+ from litellm.types.utils import Choices
22
+
23
+ if TYPE_CHECKING:
24
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
25
+ from litellm.types.utils import ModelResponse
26
+
27
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
28
+ else:
29
+ LiteLLMLoggingObj = Any
30
+
31
+
32
+ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
33
+ """
34
+ Configuration for Bedrock Moonshot AI (Kimi K2) models.
35
+
36
+ Reference:
37
+ https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/
38
+ https://platform.moonshot.ai/docs/api/chat
39
+
40
+ Supported Params for the Amazon / Moonshot models:
41
+ - `max_tokens` (integer) max tokens
42
+ - `temperature` (float) temperature for model (0-1 for Moonshot)
43
+ - `top_p` (float) top p for model
44
+ - `stream` (bool) whether to stream responses
45
+ - `tools` (list) tool definitions (supported on kimi-k2-thinking)
46
+ - `tool_choice` (str|dict) tool choice specification (supported on kimi-k2-thinking)
47
+
48
+ NOT Supported on Bedrock:
49
+ - `stop` sequences (Bedrock doesn't support stopSequences field for this model)
50
+
51
+ Note: The kimi-k2-thinking model DOES support tool calls, unlike kimi-thinking-preview.
52
+ """
53
+
54
+ def __init__(self, **kwargs):
55
+ AmazonInvokeConfig.__init__(self, **kwargs)
56
+ MoonshotChatConfig.__init__(self, **kwargs)
57
+
58
+ @property
59
+ def custom_llm_provider(self) -> Optional[str]:
60
+ return "bedrock"
61
+
62
+ def _get_model_id(self, model: str) -> str:
63
+ """
64
+ Extract the actual model ID from the LiteLLM model name.
65
+
66
+ Removes routing prefixes like:
67
+ - bedrock/invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
68
+ - invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
69
+ - moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
70
+ """
71
+ # Remove bedrock/ prefix if present
72
+ if model.startswith("bedrock/"):
73
+ model = model[8:]
74
+
75
+ # Remove invoke/ prefix if present
76
+ if model.startswith("invoke/"):
77
+ model = model[7:]
78
+
79
+ # Remove any provider prefix (e.g., moonshot/)
80
+ if "/" in model and not model.startswith("arn:"):
81
+ parts = model.split("/", 1)
82
+ if len(parts) == 2:
83
+ model = parts[1]
84
+
85
+ return model
86
+
87
+ def get_supported_openai_params(self, model: str) -> List[str]:
88
+ """
89
+ Get the supported OpenAI params for Moonshot AI models on Bedrock.
90
+
91
+ Bedrock-specific limitations:
92
+ - stopSequences field is not supported on Bedrock (unlike native Moonshot API)
93
+ - functions parameter is not supported (use tools instead)
94
+ - tool_choice doesn't support "required" value
95
+
96
+ Note: kimi-k2-thinking DOES support tool calls (unlike kimi-thinking-preview)
97
+ The parent MoonshotChatConfig class handles the kimi-thinking-preview exclusion.
98
+ """
99
+ excluded_params: List[str] = [
100
+ "functions",
101
+ "stop",
102
+ ] # Bedrock doesn't support stopSequences
103
+
104
+ base_openai_params = super(
105
+ MoonshotChatConfig, self
106
+ ).get_supported_openai_params(model=model)
107
+ final_params: List[str] = []
108
+ for param in base_openai_params:
109
+ if param not in excluded_params:
110
+ final_params.append(param)
111
+
112
+ return final_params
113
+
114
+ def map_openai_params(
115
+ self,
116
+ non_default_params: dict,
117
+ optional_params: dict,
118
+ model: str,
119
+ drop_params: bool,
120
+ ) -> dict:
121
+ """
122
+ Map OpenAI parameters to Moonshot AI parameters for Bedrock.
123
+
124
+ Handles Moonshot AI specific limitations:
125
+ - tool_choice doesn't support "required" value
126
+ - Temperature <0.3 limitation for n>1
127
+ - Temperature range is [0, 1] (not [0, 2] like OpenAI)
128
+ """
129
+ return MoonshotChatConfig.map_openai_params(
130
+ self,
131
+ non_default_params=non_default_params,
132
+ optional_params=optional_params,
133
+ model=model,
134
+ drop_params=drop_params,
135
+ )
136
+
137
+ def transform_request(
138
+ self,
139
+ model: str,
140
+ messages: List[AllMessageValues],
141
+ optional_params: dict,
142
+ litellm_params: dict,
143
+ headers: dict,
144
+ ) -> dict:
145
+ """
146
+ Transform the request for Bedrock Moonshot AI models.
147
+
148
+ Uses the Moonshot transformation logic which handles:
149
+ - Converting content lists to strings (Moonshot doesn't support list format)
150
+ - Adding tool_choice="required" message if needed
151
+ - Temperature and parameter validation
152
+
153
+ """
154
+ # Filter out AWS credentials using the existing method from BaseAWSLLM
155
+ self._get_boto_credentials_from_optional_params(optional_params, model)
156
+
157
+ # Strip routing prefixes to get the actual model ID
158
+ clean_model_id = self._get_model_id(model)
159
+
160
+ # Use Moonshot's transform_request which handles message transformation
161
+ # and tool_choice="required" workaround
162
+ return MoonshotChatConfig.transform_request(
163
+ self,
164
+ model=clean_model_id,
165
+ messages=messages,
166
+ optional_params=optional_params,
167
+ litellm_params=litellm_params,
168
+ headers=headers,
169
+ )
170
+
171
+ def _extract_reasoning_from_content(
172
+ self, content: str
173
+ ) -> tuple[Optional[str], str]:
174
+ """
175
+ Extract reasoning content from <reasoning> tags in the response.
176
+
177
+ Moonshot AI's Kimi K2 Thinking model returns reasoning in <reasoning> tags.
178
+ This method extracts that content and returns it separately.
179
+
180
+ Args:
181
+ content: The full content string from the API response
182
+
183
+ Returns:
184
+ tuple: (reasoning_content, main_content)
185
+ """
186
+ if not content:
187
+ return None, content
188
+
189
+ # Match <reasoning>...</reasoning> tags
190
+ reasoning_match = re.match(
191
+ r"<reasoning>(.*?)</reasoning>\s*(.*)", content, re.DOTALL
192
+ )
193
+
194
+ if reasoning_match:
195
+ reasoning_content = reasoning_match.group(1).strip()
196
+ main_content = reasoning_match.group(2).strip()
197
+ return reasoning_content, main_content
198
+
199
+ return None, content
200
+
201
+ def transform_response(
202
+ self,
203
+ model: str,
204
+ raw_response: httpx.Response,
205
+ model_response: "ModelResponse",
206
+ logging_obj: LiteLLMLoggingObj,
207
+ request_data: dict,
208
+ messages: List[AllMessageValues],
209
+ optional_params: dict,
210
+ litellm_params: dict,
211
+ encoding: Any,
212
+ api_key: Optional[str] = None,
213
+ json_mode: Optional[bool] = None,
214
+ ) -> "ModelResponse":
215
+ """
216
+ Transform the response from Bedrock Moonshot AI models.
217
+
218
+ Moonshot AI uses OpenAI-compatible response format, but returns reasoning
219
+ content in <reasoning> tags. This method:
220
+ 1. Calls parent class transformation
221
+ 2. Extracts reasoning content from <reasoning> tags
222
+ 3. Sets reasoning_content on the message object
223
+ """
224
+ # First, get the standard transformation
225
+ model_response = MoonshotChatConfig.transform_response(
226
+ self,
227
+ model=model,
228
+ raw_response=raw_response,
229
+ model_response=model_response,
230
+ logging_obj=logging_obj,
231
+ request_data=request_data,
232
+ messages=messages,
233
+ optional_params=optional_params,
234
+ litellm_params=litellm_params,
235
+ encoding=encoding,
236
+ api_key=api_key,
237
+ json_mode=json_mode,
238
+ )
239
+
240
+ # Extract reasoning content from <reasoning> tags
241
+ if model_response.choices and len(model_response.choices) > 0:
242
+ for choice in model_response.choices:
243
+ # Only process Choices (not StreamingChoices) which have message attribute
244
+ if (
245
+ isinstance(choice, Choices)
246
+ and choice.message
247
+ and choice.message.content
248
+ ):
249
+ (
250
+ reasoning_content,
251
+ main_content,
252
+ ) = self._extract_reasoning_from_content(choice.message.content)
253
+
254
+ if reasoning_content:
255
+ # Set the reasoning_content field
256
+ choice.message.reasoning_content = reasoning_content
257
+ # Update the main content without reasoning tags
258
+ choice.message.content = main_content
259
+
260
+ return model_response
261
+
262
+ def get_error_class(
263
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
264
+ ) -> BedrockError:
265
+ """Return the appropriate error class for Bedrock."""
266
+ return BedrockError(status_code=status_code, message=error_message)
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Handles transforming requests for `bedrock/invoke/{nova} models`
3
+
4
+ Inherits from `AmazonConverseConfig`
5
+
6
+ Nova + Invoke API Tutorial: https://docs.aws.amazon.com/nova/latest/userguide/using-invoke-api.html
7
+ """
8
+
9
+ from typing import Any, List, Optional
10
+
11
+ import httpx
12
+
13
+ from litellm.litellm_core_utils.litellm_logging import Logging
14
+ from litellm.types.llms.bedrock import BedrockInvokeNovaRequest
15
+ from litellm.types.llms.openai import AllMessageValues
16
+ from litellm.types.utils import ModelResponse
17
+
18
+ from ..converse_transformation import AmazonConverseConfig
19
+ from .base_invoke_transformation import AmazonInvokeConfig
20
+
21
+
22
+ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
23
+ """
24
+ Config for sending `nova` requests to `/bedrock/invoke/`
25
+ """
26
+
27
+ def __init__(self, **kwargs):
28
+ super().__init__(**kwargs)
29
+
30
+ def get_supported_openai_params(self, model: str) -> list:
31
+ return AmazonConverseConfig.get_supported_openai_params(self, model)
32
+
33
+ def map_openai_params(
34
+ self,
35
+ non_default_params: dict,
36
+ optional_params: dict,
37
+ model: str,
38
+ drop_params: bool,
39
+ ) -> dict:
40
+ return AmazonConverseConfig.map_openai_params(
41
+ self, non_default_params, optional_params, model, drop_params
42
+ )
43
+
44
+ def transform_request(
45
+ self,
46
+ model: str,
47
+ messages: List[AllMessageValues],
48
+ optional_params: dict,
49
+ litellm_params: dict,
50
+ headers: dict,
51
+ ) -> dict:
52
+ _transformed_nova_request = AmazonConverseConfig.transform_request(
53
+ self,
54
+ model=model,
55
+ messages=messages,
56
+ optional_params=optional_params,
57
+ litellm_params=litellm_params,
58
+ headers=headers,
59
+ )
60
+ _bedrock_invoke_nova_request = BedrockInvokeNovaRequest(
61
+ **_transformed_nova_request
62
+ )
63
+ self._remove_empty_system_messages(_bedrock_invoke_nova_request)
64
+ bedrock_invoke_nova_request = self._filter_allowed_fields(
65
+ _bedrock_invoke_nova_request
66
+ )
67
+ return bedrock_invoke_nova_request
68
+
69
+ def transform_response(
70
+ self,
71
+ model: str,
72
+ raw_response: httpx.Response,
73
+ model_response: ModelResponse,
74
+ logging_obj: Logging,
75
+ request_data: dict,
76
+ messages: List[AllMessageValues],
77
+ optional_params: dict,
78
+ litellm_params: dict,
79
+ encoding: Any,
80
+ api_key: Optional[str] = None,
81
+ json_mode: Optional[bool] = None,
82
+ ) -> ModelResponse:
83
+ return AmazonConverseConfig.transform_response(
84
+ self,
85
+ model,
86
+ raw_response,
87
+ model_response,
88
+ logging_obj,
89
+ request_data,
90
+ messages,
91
+ optional_params,
92
+ litellm_params,
93
+ encoding,
94
+ api_key,
95
+ json_mode,
96
+ )
97
+
98
+ def _filter_allowed_fields(
99
+ self, bedrock_invoke_nova_request: BedrockInvokeNovaRequest
100
+ ) -> dict:
101
+ """
102
+ Filter out fields that are not allowed in the `BedrockInvokeNovaRequest` dataclass.
103
+ """
104
+ allowed_fields = set(BedrockInvokeNovaRequest.__annotations__.keys())
105
+ return {
106
+ k: v for k, v in bedrock_invoke_nova_request.items() if k in allowed_fields
107
+ }
108
+
109
+ def _remove_empty_system_messages(
110
+ self, bedrock_invoke_nova_request: BedrockInvokeNovaRequest
111
+ ) -> None:
112
+ """
113
+ In-place remove empty `system` messages from the request.
114
+
115
+ /bedrock/invoke/ does not allow empty `system` messages.
116
+ """
117
+ _system_message = bedrock_invoke_nova_request.get("system", None)
118
+ if isinstance(_system_message, list) and len(_system_message) == 0:
119
+ bedrock_invoke_nova_request.pop("system", None)
120
+ return
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation for Bedrock imported models that use OpenAI Chat Completions format.
3
+
4
+ Use this for models imported into Bedrock that accept the OpenAI API format.
5
+ Model format: bedrock/openai/<model-id>
6
+
7
+ Example: bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123
8
+ """
9
+
10
+ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union
11
+
12
+ import httpx
13
+
14
+ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
15
+ from litellm.llms.bedrock.common_utils import BedrockError
16
+ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
17
+ from litellm.passthrough.utils import CommonUtils
18
+ from litellm.types.llms.openai import AllMessageValues
19
+
20
+ if TYPE_CHECKING:
21
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
22
+
23
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
24
+ else:
25
+ LiteLLMLoggingObj = Any
26
+
27
+
28
+ class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM):
29
+ """
30
+ Configuration for Bedrock imported models that use OpenAI Chat Completions format.
31
+
32
+ This class handles the transformation of requests and responses for Bedrock
33
+ imported models that accept the OpenAI API format directly.
34
+
35
+ Inherits from OpenAIGPTConfig to leverage standard OpenAI parameter handling
36
+ and response transformation, while adding Bedrock-specific URL generation
37
+ and AWS request signing.
38
+
39
+ Usage:
40
+ model = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123"
41
+ """
42
+
43
+ def __init__(self, **kwargs):
44
+ OpenAIGPTConfig.__init__(self, **kwargs)
45
+ BaseAWSLLM.__init__(self, **kwargs)
46
+
47
+ @property
48
+ def custom_llm_provider(self) -> Optional[str]:
49
+ return "bedrock"
50
+
51
+ def _get_openai_model_id(self, model: str) -> str:
52
+ """
53
+ Extract the actual model ID from the LiteLLM model name.
54
+
55
+ Input format: bedrock/openai/<model-id>
56
+ Returns: <model-id>
57
+ """
58
+ # Remove bedrock/ prefix if present
59
+ if model.startswith("bedrock/"):
60
+ model = model[8:]
61
+
62
+ # Remove openai/ prefix
63
+ if model.startswith("openai/"):
64
+ model = model[7:]
65
+
66
+ return model
67
+
68
+ def get_complete_url(
69
+ self,
70
+ api_base: Optional[str],
71
+ api_key: Optional[str],
72
+ model: str,
73
+ optional_params: dict,
74
+ litellm_params: dict,
75
+ stream: Optional[bool] = None,
76
+ ) -> str:
77
+ """
78
+ Get the complete URL for the Bedrock invoke endpoint.
79
+
80
+ Uses the standard Bedrock invoke endpoint format.
81
+ """
82
+ model_id = self._get_openai_model_id(model)
83
+
84
+ # Get AWS region
85
+ aws_region_name = self._get_aws_region_name(
86
+ optional_params=optional_params, model=model
87
+ )
88
+
89
+ # Get runtime endpoint
90
+ aws_bedrock_runtime_endpoint = optional_params.get(
91
+ "aws_bedrock_runtime_endpoint", None
92
+ )
93
+ endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
94
+ api_base=api_base,
95
+ aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
96
+ aws_region_name=aws_region_name,
97
+ )
98
+
99
+ # Encode model ID for ARNs (e.g., :imported-model/ -> :imported-model%2F)
100
+ model_id = CommonUtils.encode_bedrock_runtime_modelid_arn(model_id)
101
+
102
+ # Build the invoke URL
103
+ if stream:
104
+ endpoint_url = (
105
+ f"{endpoint_url}/model/{model_id}/invoke-with-response-stream"
106
+ )
107
+ else:
108
+ endpoint_url = f"{endpoint_url}/model/{model_id}/invoke"
109
+
110
+ return endpoint_url
111
+
112
+ def sign_request(
113
+ self,
114
+ headers: dict,
115
+ optional_params: dict,
116
+ request_data: dict,
117
+ api_base: str,
118
+ api_key: Optional[str] = None,
119
+ model: Optional[str] = None,
120
+ stream: Optional[bool] = None,
121
+ fake_stream: Optional[bool] = None,
122
+ ) -> Tuple[dict, Optional[bytes]]:
123
+ """
124
+ Sign the request using AWS Signature Version 4.
125
+ """
126
+ return self._sign_request(
127
+ service_name="bedrock",
128
+ headers=headers,
129
+ optional_params=optional_params,
130
+ request_data=request_data,
131
+ api_base=api_base,
132
+ api_key=api_key,
133
+ model=model,
134
+ stream=stream,
135
+ fake_stream=fake_stream,
136
+ )
137
+
138
+ def transform_request(
139
+ self,
140
+ model: str,
141
+ messages: List[AllMessageValues],
142
+ optional_params: dict,
143
+ litellm_params: dict,
144
+ headers: dict,
145
+ ) -> dict:
146
+ """
147
+ Transform the request to OpenAI Chat Completions format for Bedrock imported models.
148
+
149
+ Removes AWS-specific params and stream param (handled separately in URL),
150
+ then delegates to parent class for standard OpenAI request transformation.
151
+ """
152
+ # Remove stream from optional_params as it's handled separately in URL
153
+ optional_params.pop("stream", None)
154
+
155
+ # Remove AWS-specific params that shouldn't be in the request body
156
+ inference_params = {
157
+ k: v
158
+ for k, v in optional_params.items()
159
+ if k not in self.aws_authentication_params
160
+ }
161
+
162
+ # Use parent class transform_request for OpenAI format
163
+ return super().transform_request(
164
+ model=self._get_openai_model_id(model),
165
+ messages=messages,
166
+ optional_params=inference_params,
167
+ litellm_params=litellm_params,
168
+ headers=headers,
169
+ )
170
+
171
+ def validate_environment(
172
+ self,
173
+ headers: dict,
174
+ model: str,
175
+ messages: List[AllMessageValues],
176
+ optional_params: dict,
177
+ litellm_params: dict,
178
+ api_key: Optional[str] = None,
179
+ api_base: Optional[str] = None,
180
+ ) -> dict:
181
+ """
182
+ Validate the environment and return headers.
183
+
184
+ For Bedrock, we don't need Bearer token auth since we use AWS SigV4.
185
+ """
186
+ return headers
187
+
188
+ def get_error_class(
189
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
190
+ ) -> BedrockError:
191
+ """Return the appropriate error class for Bedrock."""
192
+ return BedrockError(status_code=status_code, message=error_message)
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Handles transforming requests for `bedrock/invoke/{qwen2} models`
3
+
4
+ Inherits from `AmazonQwen3Config` since Qwen2 and Qwen3 architectures are mostly similar.
5
+ The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field.
6
+
7
+ Qwen2 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
8
+ """
9
+
10
+ from typing import Any, List, Optional
11
+
12
+ import httpx
13
+
14
+ from litellm.llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import (
15
+ AmazonQwen3Config,
16
+ )
17
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
18
+ LiteLLMLoggingObj,
19
+ )
20
+ from litellm.types.llms.openai import AllMessageValues
21
+ from litellm.types.utils import ModelResponse, Usage
22
+
23
+
24
+ class AmazonQwen2Config(AmazonQwen3Config):
25
+ """
26
+ Config for sending `qwen2` requests to `/bedrock/invoke/`
27
+
28
+ Inherits from AmazonQwen3Config since Qwen2 and Qwen3 architectures are mostly similar.
29
+ The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field.
30
+
31
+ Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
32
+ """
33
+
34
+ def transform_response(
35
+ self,
36
+ model: str,
37
+ raw_response: httpx.Response,
38
+ model_response: ModelResponse,
39
+ logging_obj: LiteLLMLoggingObj,
40
+ request_data: dict,
41
+ messages: List[AllMessageValues],
42
+ optional_params: dict,
43
+ litellm_params: dict,
44
+ encoding: Any,
45
+ api_key: Optional[str] = None,
46
+ json_mode: Optional[bool] = None,
47
+ ) -> ModelResponse:
48
+ """
49
+ Transform Qwen2 Bedrock response to OpenAI format
50
+
51
+ Qwen2 uses "text" field, but we also support "generation" field for compatibility.
52
+ """
53
+ try:
54
+ if hasattr(raw_response, "json"):
55
+ response_data = raw_response.json()
56
+ else:
57
+ response_data = raw_response
58
+
59
+ # Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility
60
+ generated_text = response_data.get("generation", "") or response_data.get(
61
+ "text", ""
62
+ )
63
+
64
+ # Clean up the response (remove assistant start token if present)
65
+ if generated_text.startswith("<|im_start|>assistant\n"):
66
+ generated_text = generated_text[len("<|im_start|>assistant\n") :]
67
+ if generated_text.endswith("<|im_end|>"):
68
+ generated_text = generated_text[: -len("<|im_end|>")]
69
+
70
+ # Set the content in the existing model_response structure
71
+ if hasattr(model_response, "choices") and len(model_response.choices) > 0:
72
+ choice = model_response.choices[0]
73
+ choice.message.content = generated_text
74
+ choice.finish_reason = "stop"
75
+
76
+ # Set usage information if available in response
77
+ if "usage" in response_data:
78
+ usage_data = response_data["usage"]
79
+ setattr(
80
+ model_response,
81
+ "usage",
82
+ Usage(
83
+ prompt_tokens=usage_data.get("prompt_tokens", 0),
84
+ completion_tokens=usage_data.get("completion_tokens", 0),
85
+ total_tokens=usage_data.get("total_tokens", 0),
86
+ ),
87
+ )
88
+
89
+ return model_response
90
+
91
+ except Exception as e:
92
+ if logging_obj:
93
+ logging_obj.post_call(
94
+ input=messages,
95
+ api_key=api_key,
96
+ original_response=raw_response,
97
+ additional_args={"error": str(e)},
98
+ )
99
+ raise e
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py ADDED
@@ -0,0 +1,225 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Handles transforming requests for `bedrock/invoke/{qwen3} models`
3
+
4
+ Inherits from `AmazonInvokeConfig`
5
+
6
+ Qwen3 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
7
+ """
8
+
9
+ from typing import Any, List, Optional
10
+
11
+ import httpx
12
+
13
+ from litellm.llms.base_llm.chat.transformation import BaseConfig
14
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
15
+ AmazonInvokeConfig,
16
+ LiteLLMLoggingObj,
17
+ )
18
+ from litellm.types.llms.openai import AllMessageValues
19
+ from litellm.types.utils import ModelResponse, Usage
20
+
21
+
22
+ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
23
+ """
24
+ Config for sending `qwen3` requests to `/bedrock/invoke/`
25
+
26
+ Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
27
+ """
28
+
29
+ max_tokens: Optional[int] = None
30
+ temperature: Optional[float] = None
31
+ top_p: Optional[float] = None
32
+ top_k: Optional[int] = None
33
+ stop: Optional[List[str]] = None
34
+
35
+ def __init__(
36
+ self,
37
+ max_tokens: Optional[int] = None,
38
+ temperature: Optional[float] = None,
39
+ top_p: Optional[float] = None,
40
+ top_k: Optional[int] = None,
41
+ stop: Optional[List[str]] = None,
42
+ ) -> None:
43
+ locals_ = locals().copy()
44
+ for key, value in locals_.items():
45
+ if key != "self" and value is not None:
46
+ setattr(self.__class__, key, value)
47
+ AmazonInvokeConfig.__init__(self)
48
+
49
+ def get_supported_openai_params(self, model: str) -> List[str]:
50
+ return [
51
+ "max_tokens",
52
+ "temperature",
53
+ "top_p",
54
+ "top_k",
55
+ "stop",
56
+ "stream",
57
+ ]
58
+
59
+ def map_openai_params(
60
+ self,
61
+ non_default_params: dict,
62
+ optional_params: dict,
63
+ model: str,
64
+ drop_params: bool,
65
+ ) -> dict:
66
+ for k, v in non_default_params.items():
67
+ if k == "max_tokens":
68
+ optional_params["max_tokens"] = v
69
+ if k == "temperature":
70
+ optional_params["temperature"] = v
71
+ if k == "top_p":
72
+ optional_params["top_p"] = v
73
+ if k == "top_k":
74
+ optional_params["top_k"] = v
75
+ if k == "stop":
76
+ optional_params["stop"] = v
77
+ if k == "stream":
78
+ optional_params["stream"] = v
79
+ return optional_params
80
+
81
+ def transform_request(
82
+ self,
83
+ model: str,
84
+ messages: List[AllMessageValues],
85
+ optional_params: dict,
86
+ litellm_params: dict,
87
+ headers: dict,
88
+ ) -> dict:
89
+ """
90
+ Transform OpenAI format to Qwen3 Bedrock invoke format
91
+ """
92
+ # Convert messages to prompt format
93
+ prompt = self._convert_messages_to_prompt(messages)
94
+
95
+ # Build the request body
96
+ request_body = {
97
+ "prompt": prompt,
98
+ }
99
+
100
+ # Add optional parameters
101
+ if "max_tokens" in optional_params:
102
+ request_body["max_gen_len"] = optional_params["max_tokens"]
103
+ if "temperature" in optional_params:
104
+ request_body["temperature"] = optional_params["temperature"]
105
+ if "top_p" in optional_params:
106
+ request_body["top_p"] = optional_params["top_p"]
107
+ if "top_k" in optional_params:
108
+ request_body["top_k"] = optional_params["top_k"]
109
+ if "stop" in optional_params:
110
+ request_body["stop"] = optional_params["stop"]
111
+
112
+ return request_body
113
+
114
+ def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str:
115
+ """
116
+ Convert OpenAI messages format to Qwen3 prompt format
117
+ Supports tool calls, multimodal content, and various message types
118
+ """
119
+ prompt_parts = []
120
+
121
+ for message in messages:
122
+ role = message.get("role", "")
123
+ content = message.get("content", "")
124
+ tool_calls = message.get("tool_calls", [])
125
+
126
+ if role == "system":
127
+ prompt_parts.append(f"<|im_start|>system\n{content}<|im_end|>")
128
+ elif role == "user":
129
+ # Handle multimodal content
130
+ if isinstance(content, list):
131
+ text_content = []
132
+ for item in content:
133
+ if item.get("type") == "text":
134
+ text_content.append(item.get("text", ""))
135
+ elif item.get("type") == "image_url":
136
+ # For Qwen3, we can include image placeholders
137
+ text_content.append(
138
+ "<|vision_start|><|image_pad|><|vision_end|>"
139
+ )
140
+ content = "".join(text_content)
141
+ prompt_parts.append(f"<|im_start|>user\n{content}<|im_end|>")
142
+ elif role == "assistant":
143
+ if tool_calls and isinstance(tool_calls, list):
144
+ # Handle tool calls
145
+ for tool_call in tool_calls:
146
+ function_name = tool_call.get("function", {}).get("name", "")
147
+ function_args = tool_call.get("function", {}).get(
148
+ "arguments", ""
149
+ )
150
+ prompt_parts.append(
151
+ f'<|im_start|>assistant\n<tool_call>\n{{"name": "{function_name}", "arguments": "{function_args}"}}\n</tool_call><|im_end|>'
152
+ )
153
+ else:
154
+ prompt_parts.append(f"<|im_start|>assistant\n{content}<|im_end|>")
155
+ elif role == "tool":
156
+ # Handle tool responses
157
+ prompt_parts.append(f"<|im_start|>tool\n{content}<|im_end|>")
158
+
159
+ # Add assistant start token for response generation
160
+ prompt_parts.append("<|im_start|>assistant\n")
161
+
162
+ return "\n".join(prompt_parts)
163
+
164
+ def transform_response(
165
+ self,
166
+ model: str,
167
+ raw_response: httpx.Response,
168
+ model_response: ModelResponse,
169
+ logging_obj: LiteLLMLoggingObj,
170
+ request_data: dict,
171
+ messages: List[AllMessageValues],
172
+ optional_params: dict,
173
+ litellm_params: dict,
174
+ encoding: Any,
175
+ api_key: Optional[str] = None,
176
+ json_mode: Optional[bool] = None,
177
+ ) -> ModelResponse:
178
+ """
179
+ Transform Qwen3 Bedrock response to OpenAI format
180
+ """
181
+ try:
182
+ if hasattr(raw_response, "json"):
183
+ response_data = raw_response.json()
184
+ else:
185
+ response_data = raw_response
186
+
187
+ # Extract the generated text - Qwen3 uses "generation" field
188
+ generated_text = response_data.get("generation", "")
189
+
190
+ # Clean up the response (remove assistant start token if present)
191
+ if generated_text.startswith("<|im_start|>assistant\n"):
192
+ generated_text = generated_text[len("<|im_start|>assistant\n") :]
193
+ if generated_text.endswith("<|im_end|>"):
194
+ generated_text = generated_text[: -len("<|im_end|>")]
195
+
196
+ # Set the content in the existing model_response structure
197
+ if hasattr(model_response, "choices") and len(model_response.choices) > 0:
198
+ choice = model_response.choices[0]
199
+ choice.message.content = generated_text
200
+ choice.finish_reason = "stop"
201
+
202
+ # Set usage information if available in response
203
+ if "usage" in response_data:
204
+ usage_data = response_data["usage"]
205
+ setattr(
206
+ model_response,
207
+ "usage",
208
+ Usage(
209
+ prompt_tokens=usage_data.get("prompt_tokens", 0),
210
+ completion_tokens=usage_data.get("completion_tokens", 0),
211
+ total_tokens=usage_data.get("total_tokens", 0),
212
+ ),
213
+ )
214
+
215
+ return model_response
216
+
217
+ except Exception as e:
218
+ if logging_obj:
219
+ logging_obj.post_call(
220
+ input=messages,
221
+ api_key=api_key,
222
+ original_response=raw_response,
223
+ additional_args={"error": str(e)},
224
+ )
225
+ raise e
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import types
3
+ from typing import List, Optional, Union
4
+
5
+ import litellm
6
+ from litellm.llms.base_llm.chat.transformation import BaseConfig
7
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
8
+ AmazonInvokeConfig,
9
+ )
10
+
11
+
12
+ class AmazonTitanConfig(AmazonInvokeConfig, BaseConfig):
13
+ """
14
+ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=titan-text-express-v1
15
+
16
+ Supported Params for the Amazon Titan models:
17
+
18
+ - `maxTokenCount` (integer) max tokens,
19
+ - `stopSequences` (string[]) list of stop sequence strings
20
+ - `temperature` (float) temperature for model,
21
+ - `topP` (int) top p for model
22
+ """
23
+
24
+ maxTokenCount: Optional[int] = None
25
+ stopSequences: Optional[list] = None
26
+ temperature: Optional[float] = None
27
+ topP: Optional[int] = None
28
+
29
+ def __init__(
30
+ self,
31
+ maxTokenCount: Optional[int] = None,
32
+ stopSequences: Optional[list] = None,
33
+ temperature: Optional[float] = None,
34
+ topP: Optional[int] = None,
35
+ ) -> None:
36
+ locals_ = locals().copy()
37
+ for key, value in locals_.items():
38
+ if key != "self" and value is not None:
39
+ setattr(self.__class__, key, value)
40
+
41
+ AmazonInvokeConfig.__init__(self)
42
+
43
+ @classmethod
44
+ def get_config(cls):
45
+ return {
46
+ k: v
47
+ for k, v in cls.__dict__.items()
48
+ if not k.startswith("__")
49
+ and not k.startswith("_abc")
50
+ and not isinstance(
51
+ v,
52
+ (
53
+ types.FunctionType,
54
+ types.BuiltinFunctionType,
55
+ classmethod,
56
+ staticmethod,
57
+ ),
58
+ )
59
+ and v is not None
60
+ }
61
+
62
+ def _map_and_modify_arg(
63
+ self,
64
+ supported_params: dict,
65
+ provider: str,
66
+ model: str,
67
+ stop: Union[List[str], str],
68
+ ):
69
+ """
70
+ filter params to fit the required provider format, drop those that don't fit if user sets `litellm.drop_params = True`.
71
+ """
72
+ filtered_stop = None
73
+ if "stop" in supported_params and litellm.drop_params:
74
+ if provider == "bedrock" and "amazon" in model:
75
+ filtered_stop = []
76
+ if isinstance(stop, list):
77
+ for s in stop:
78
+ if re.match(r"^(\|+|User:)$", s):
79
+ filtered_stop.append(s)
80
+ if filtered_stop is not None:
81
+ supported_params["stop"] = filtered_stop
82
+
83
+ return supported_params
84
+
85
+ def get_supported_openai_params(self, model: str) -> List[str]:
86
+ return [
87
+ "max_tokens",
88
+ "max_completion_tokens",
89
+ "stop",
90
+ "temperature",
91
+ "top_p",
92
+ "stream",
93
+ ]
94
+
95
+ def map_openai_params(
96
+ self,
97
+ non_default_params: dict,
98
+ optional_params: dict,
99
+ model: str,
100
+ drop_params: bool,
101
+ ) -> dict:
102
+ for k, v in non_default_params.items():
103
+ if k == "max_tokens" or k == "max_completion_tokens":
104
+ optional_params["maxTokenCount"] = v
105
+ if k == "temperature":
106
+ optional_params["temperature"] = v
107
+ if k == "stop":
108
+ filtered_stop = self._map_and_modify_arg(
109
+ {"stop": v}, provider="bedrock", model=model, stop=v
110
+ )
111
+ optional_params["stopSequences"] = filtered_stop["stop"]
112
+ if k == "top_p":
113
+ optional_params["topP"] = v
114
+ if k == "stream":
115
+ optional_params["stream"] = v
116
+ return optional_params
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py ADDED
@@ -0,0 +1,280 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transforms OpenAI-style requests into TwelveLabs Pegasus 1.2 requests for Bedrock.
3
+
4
+ Reference:
5
+ https://docs.twelvelabs.io/docs/models/pegasus
6
+ """
7
+
8
+ import json
9
+ import time
10
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional
11
+
12
+ import httpx
13
+
14
+ import litellm
15
+ from litellm._logging import verbose_logger
16
+ from litellm.litellm_core_utils.core_helpers import map_finish_reason
17
+ from litellm.llms.base_llm.base_utils import type_to_response_format_param
18
+ from litellm.llms.base_llm.chat.transformation import BaseConfig
19
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
20
+ AmazonInvokeConfig,
21
+ )
22
+ from litellm.llms.bedrock.common_utils import BedrockError
23
+ from litellm.types.llms.openai import AllMessageValues
24
+ from litellm.types.utils import ModelResponse, Usage
25
+ from litellm.utils import get_base64_str
26
+
27
+ if TYPE_CHECKING:
28
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
29
+
30
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
31
+ else:
32
+ LiteLLMLoggingObj = Any
33
+
34
+
35
+ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
36
+ """
37
+ Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
38
+ `twelvelabs.pegasus-1-2-v1:0`.
39
+
40
+ Pegasus 1.2 requires an `inputPrompt` and a `mediaSource` that either references
41
+ an S3 object or a base64-encoded clip. Optional OpenAI params (temperature,
42
+ response_format, max_tokens) are translated to the TwelveLabs schema.
43
+ """
44
+
45
+ def get_supported_openai_params(self, model: str) -> List[str]:
46
+ return [
47
+ "max_tokens",
48
+ "max_completion_tokens",
49
+ "temperature",
50
+ "response_format",
51
+ ]
52
+
53
+ def map_openai_params(
54
+ self,
55
+ non_default_params: dict,
56
+ optional_params: dict,
57
+ model: str,
58
+ drop_params: bool,
59
+ ) -> dict:
60
+ for param, value in non_default_params.items():
61
+ if param in {"max_tokens", "max_completion_tokens"}:
62
+ optional_params["maxOutputTokens"] = value
63
+ if param == "temperature":
64
+ optional_params["temperature"] = value
65
+ if param == "response_format":
66
+ optional_params["responseFormat"] = self._normalize_response_format(
67
+ value
68
+ )
69
+ return optional_params
70
+
71
+ def _normalize_response_format(self, value: Any) -> Any:
72
+ """Normalize response_format to TwelveLabs format.
73
+
74
+ TwelveLabs expects:
75
+ {
76
+ "jsonSchema": {...}
77
+ }
78
+
79
+ But OpenAI format is:
80
+ {
81
+ "type": "json_schema",
82
+ "json_schema": {
83
+ "name": "...",
84
+ "schema": {...}
85
+ }
86
+ }
87
+ """
88
+ if isinstance(value, dict):
89
+ # If it has json_schema field, extract and transform it
90
+ if "json_schema" in value:
91
+ json_schema = value["json_schema"]
92
+ # Extract the schema if nested
93
+ if isinstance(json_schema, dict) and "schema" in json_schema:
94
+ return {"jsonSchema": json_schema["schema"]}
95
+ # Otherwise use json_schema directly
96
+ return {"jsonSchema": json_schema}
97
+ # If it already has jsonSchema, return as is
98
+ if "jsonSchema" in value:
99
+ return value
100
+ # Otherwise return the dict as is
101
+ return value
102
+ return type_to_response_format_param(response_format=value) or value
103
+
104
+ def transform_request(
105
+ self,
106
+ model: str,
107
+ messages: List[AllMessageValues],
108
+ optional_params: dict,
109
+ litellm_params: dict,
110
+ headers: dict,
111
+ ) -> dict:
112
+ input_prompt = self._convert_messages_to_prompt(messages=messages)
113
+ request_data: Dict[str, Any] = {"inputPrompt": input_prompt}
114
+
115
+ media_source = self._build_media_source(optional_params)
116
+ if media_source is not None:
117
+ request_data["mediaSource"] = media_source
118
+
119
+ # Handle temperature and maxOutputTokens
120
+ for key in ("temperature", "maxOutputTokens"):
121
+ if key in optional_params:
122
+ request_data[key] = optional_params.get(key)
123
+
124
+ # Handle responseFormat - transform to TwelveLabs format
125
+ if "responseFormat" in optional_params:
126
+ response_format = optional_params["responseFormat"]
127
+ transformed_format = self._normalize_response_format(response_format)
128
+ if transformed_format:
129
+ request_data["responseFormat"] = transformed_format
130
+
131
+ return request_data
132
+
133
+ def _build_media_source(self, optional_params: dict) -> Optional[dict]:
134
+ direct_source = optional_params.get("mediaSource") or optional_params.get(
135
+ "media_source"
136
+ )
137
+ if isinstance(direct_source, dict):
138
+ return direct_source
139
+
140
+ base64_input = optional_params.get("video_base64") or optional_params.get(
141
+ "base64_string"
142
+ )
143
+ if base64_input:
144
+ return {"base64String": get_base64_str(base64_input)}
145
+
146
+ s3_uri = (
147
+ optional_params.get("video_s3_uri")
148
+ or optional_params.get("s3_uri")
149
+ or optional_params.get("media_source_s3_uri")
150
+ )
151
+ if s3_uri:
152
+ s3_location = {"uri": s3_uri}
153
+ bucket_owner = (
154
+ optional_params.get("video_s3_bucket_owner")
155
+ or optional_params.get("s3_bucket_owner")
156
+ or optional_params.get("media_source_bucket_owner")
157
+ )
158
+ if bucket_owner:
159
+ s3_location["bucketOwner"] = bucket_owner
160
+ return {"s3Location": s3_location}
161
+ return None
162
+
163
+ def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str:
164
+ prompt_parts: List[str] = []
165
+ for message in messages:
166
+ role = message.get("role", "user")
167
+ content = message.get("content", "")
168
+ if isinstance(content, list):
169
+ text_fragments = []
170
+ for item in content:
171
+ if isinstance(item, dict):
172
+ item_type = item.get("type")
173
+ if item_type == "text":
174
+ text_fragments.append(item.get("text", ""))
175
+ elif item_type == "image_url":
176
+ text_fragments.append("<image>")
177
+ elif item_type == "video_url":
178
+ text_fragments.append("<video>")
179
+ elif item_type == "audio_url":
180
+ text_fragments.append("<audio>")
181
+ elif isinstance(item, str):
182
+ text_fragments.append(item)
183
+ content = " ".join(text_fragments)
184
+ prompt_parts.append(f"{role}: {content}")
185
+ return "\n".join(part for part in prompt_parts if part).strip()
186
+
187
+ def transform_response(
188
+ self,
189
+ model: str,
190
+ raw_response: httpx.Response,
191
+ model_response: ModelResponse,
192
+ logging_obj: LiteLLMLoggingObj,
193
+ request_data: dict,
194
+ messages: List[AllMessageValues],
195
+ optional_params: dict,
196
+ litellm_params: dict,
197
+ encoding: Any,
198
+ api_key: Optional[str] = None,
199
+ json_mode: Optional[bool] = None,
200
+ ) -> ModelResponse:
201
+ """
202
+ Transform TwelveLabs Pegasus response to LiteLLM format.
203
+
204
+ TwelveLabs response format:
205
+ {
206
+ "message": "...",
207
+ "finishReason": "stop" | "length"
208
+ }
209
+
210
+ LiteLLM format:
211
+ ModelResponse with choices[0].message.content and finish_reason
212
+ """
213
+ try:
214
+ completion_response = raw_response.json()
215
+ except Exception as e:
216
+ raise BedrockError(
217
+ message=f"Error parsing response: {raw_response.text}, error: {str(e)}",
218
+ status_code=raw_response.status_code,
219
+ )
220
+
221
+ verbose_logger.debug(
222
+ "twelvelabs pegasus response: %s",
223
+ json.dumps(completion_response, indent=4, default=str),
224
+ )
225
+
226
+ # Extract message content
227
+ message_content = completion_response.get("message", "")
228
+
229
+ # Extract finish reason and map to LiteLLM format
230
+ finish_reason_raw = completion_response.get("finishReason", "stop")
231
+ finish_reason = map_finish_reason(finish_reason_raw)
232
+
233
+ # Set the response content
234
+ try:
235
+ if (
236
+ message_content
237
+ and hasattr(model_response.choices[0], "message")
238
+ and getattr(model_response.choices[0].message, "tool_calls", None)
239
+ is None
240
+ ):
241
+ model_response.choices[0].message.content = message_content # type: ignore
242
+ model_response.choices[0].finish_reason = finish_reason
243
+ else:
244
+ raise Exception("Unable to set message content")
245
+ except Exception as e:
246
+ raise BedrockError(
247
+ message=f"Error setting response content: {str(e)}. Response: {completion_response}",
248
+ status_code=raw_response.status_code,
249
+ )
250
+
251
+ # Calculate usage from headers
252
+ bedrock_input_tokens = raw_response.headers.get(
253
+ "x-amzn-bedrock-input-token-count", None
254
+ )
255
+ bedrock_output_tokens = raw_response.headers.get(
256
+ "x-amzn-bedrock-output-token-count", None
257
+ )
258
+
259
+ prompt_tokens = int(
260
+ bedrock_input_tokens or litellm.token_counter(messages=messages)
261
+ )
262
+
263
+ completion_tokens = int(
264
+ bedrock_output_tokens
265
+ or litellm.token_counter(
266
+ text=model_response.choices[0].message.content, # type: ignore
267
+ count_response_tokens=True,
268
+ )
269
+ )
270
+
271
+ model_response.created = int(time.time())
272
+ model_response.model = model
273
+ usage = Usage(
274
+ prompt_tokens=prompt_tokens,
275
+ completion_tokens=completion_tokens,
276
+ total_tokens=prompt_tokens + completion_tokens,
277
+ )
278
+ setattr(model_response, "usage", usage)
279
+
280
+ return model_response
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import types
2
+ from typing import Optional
3
+
4
+ import litellm
5
+
6
+ from .base_invoke_transformation import AmazonInvokeConfig
7
+
8
+
9
+ class AmazonAnthropicConfig(AmazonInvokeConfig):
10
+ """
11
+ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=claude
12
+
13
+ Supported Params for the Amazon / Anthropic models:
14
+
15
+ - `max_tokens_to_sample` (integer) max tokens,
16
+ - `temperature` (float) model temperature,
17
+ - `top_k` (integer) top k,
18
+ - `top_p` (integer) top p,
19
+ - `stop_sequences` (string[]) list of stop sequences - e.g. ["\\n\\nHuman:"],
20
+ - `anthropic_version` (string) version of anthropic for bedrock - e.g. "bedrock-2023-05-31"
21
+ """
22
+
23
+ max_tokens_to_sample: Optional[int] = litellm.max_tokens
24
+ stop_sequences: Optional[list] = None
25
+ temperature: Optional[float] = None
26
+ top_k: Optional[int] = None
27
+ top_p: Optional[int] = None
28
+ anthropic_version: Optional[str] = None
29
+
30
+ def __init__(
31
+ self,
32
+ max_tokens_to_sample: Optional[int] = None,
33
+ stop_sequences: Optional[list] = None,
34
+ temperature: Optional[float] = None,
35
+ top_k: Optional[int] = None,
36
+ top_p: Optional[int] = None,
37
+ anthropic_version: Optional[str] = None,
38
+ ) -> None:
39
+ locals_ = locals().copy()
40
+ for key, value in locals_.items():
41
+ if key != "self" and value is not None:
42
+ setattr(self.__class__, key, value)
43
+
44
+ @classmethod
45
+ def get_config(cls):
46
+ return {
47
+ k: v
48
+ for k, v in cls.__dict__.items()
49
+ if not k.startswith("__")
50
+ and not isinstance(
51
+ v,
52
+ (
53
+ types.FunctionType,
54
+ types.BuiltinFunctionType,
55
+ classmethod,
56
+ staticmethod,
57
+ ),
58
+ )
59
+ and v is not None
60
+ }
61
+
62
+ @staticmethod
63
+ def get_legacy_anthropic_model_names():
64
+ return [
65
+ "anthropic.claude-v2",
66
+ "anthropic.claude-instant-v1",
67
+ "anthropic.claude-v2:1",
68
+ ]
69
+
70
+ def get_supported_openai_params(self, model: str):
71
+ return [
72
+ "max_tokens",
73
+ "max_completion_tokens",
74
+ "temperature",
75
+ "stop",
76
+ "top_p",
77
+ "stream",
78
+ ]
79
+
80
+ def map_openai_params(
81
+ self,
82
+ non_default_params: dict,
83
+ optional_params: dict,
84
+ model: str,
85
+ drop_params: bool,
86
+ ):
87
+ for param, value in non_default_params.items():
88
+ if param == "max_tokens" or param == "max_completion_tokens":
89
+ optional_params["max_tokens_to_sample"] = value
90
+ if param == "temperature":
91
+ optional_params["temperature"] = value
92
+ if param == "top_p":
93
+ optional_params["top_p"] = value
94
+ if param == "stop":
95
+ optional_params["stop_sequences"] = value
96
+ if param == "stream" and value is True:
97
+ optional_params["stream"] = value
98
+ return optional_params
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py ADDED
@@ -0,0 +1,206 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import TYPE_CHECKING, Any, List, Optional
2
+
3
+ import httpx
4
+
5
+ from litellm.llms.anthropic.chat.transformation import AnthropicConfig
6
+ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
7
+ AmazonInvokeConfig,
8
+ )
9
+ from litellm.llms.bedrock.common_utils import (
10
+ get_anthropic_beta_from_headers,
11
+ remove_custom_field_from_tools,
12
+ )
13
+ from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
14
+ from litellm.types.llms.openai import AllMessageValues
15
+ from litellm.types.utils import ModelResponse
16
+
17
+ if TYPE_CHECKING:
18
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
19
+
20
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
21
+ else:
22
+ LiteLLMLoggingObj = Any
23
+
24
+
25
+ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
26
+ """
27
+ Reference:
28
+ https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=claude
29
+ https://docs.anthropic.com/claude/docs/models-overview#model-comparison
30
+ https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
31
+
32
+ Supported Params for the Amazon / Anthropic Claude models (Claude 3, Claude 4, etc.):
33
+ Supports anthropic_beta parameter for beta features like:
34
+ - computer-use-2025-01-24 (Claude 3.7 Sonnet)
35
+ - computer-use-2024-10-22 (Claude 3.5 Sonnet v2)
36
+ - token-efficient-tools-2025-02-19 (Claude 3.7 Sonnet)
37
+ - interleaved-thinking-2025-05-14 (Claude 4 models)
38
+ - output-128k-2025-02-19 (Claude 3.7 Sonnet)
39
+ - dev-full-thinking-2025-05-14 (Claude 4 models)
40
+ - context-1m-2025-08-07 (Claude Sonnet 4)
41
+ """
42
+
43
+ anthropic_version: str = "bedrock-2023-05-31"
44
+
45
+ @property
46
+ def custom_llm_provider(self) -> Optional[str]:
47
+ return "bedrock"
48
+
49
+ def get_supported_openai_params(self, model: str) -> List[str]:
50
+ return AnthropicConfig.get_supported_openai_params(self, model)
51
+
52
+ def map_openai_params(
53
+ self,
54
+ non_default_params: dict,
55
+ optional_params: dict,
56
+ model: str,
57
+ drop_params: bool,
58
+ ) -> dict:
59
+ # Force tool-based structured outputs for Bedrock Invoke
60
+ # (similar to VertexAI fix in #19201)
61
+ # Bedrock Invoke doesn't support output_format parameter
62
+ original_model = model
63
+ if "response_format" in non_default_params:
64
+ # Use a model name that forces tool-based approach
65
+ model = "claude-3-sonnet-20240229"
66
+
67
+ optional_params = AnthropicConfig.map_openai_params(
68
+ self,
69
+ non_default_params,
70
+ optional_params,
71
+ model,
72
+ drop_params,
73
+ )
74
+
75
+ # Restore original model name
76
+ model = original_model
77
+
78
+ return optional_params
79
+
80
+ def transform_request(
81
+ self,
82
+ model: str,
83
+ messages: List[AllMessageValues],
84
+ optional_params: dict,
85
+ litellm_params: dict,
86
+ headers: dict,
87
+ ) -> dict:
88
+ # Filter out AWS authentication parameters before passing to Anthropic transformation
89
+ # AWS params should only be used for signing requests, not included in request body
90
+ filtered_params = {
91
+ k: v
92
+ for k, v in optional_params.items()
93
+ if k not in self.aws_authentication_params
94
+ }
95
+ filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params)
96
+
97
+ _anthropic_request = AnthropicConfig.transform_request(
98
+ self,
99
+ model=model,
100
+ messages=messages,
101
+ optional_params=filtered_params,
102
+ litellm_params=litellm_params,
103
+ headers=headers,
104
+ )
105
+
106
+ _anthropic_request.pop("model", None)
107
+ _anthropic_request.pop("stream", None)
108
+ # Bedrock Invoke doesn't support output_format parameter
109
+ _anthropic_request.pop("output_format", None)
110
+ # Bedrock Invoke doesn't support output_config parameter
111
+ # Fixes: https://github.com/BerriAI/litellm/issues/22797
112
+ _anthropic_request.pop("output_config", None)
113
+ if "anthropic_version" not in _anthropic_request:
114
+ _anthropic_request["anthropic_version"] = self.anthropic_version
115
+
116
+ # Remove `custom` field from tools (Bedrock doesn't support it)
117
+ # Claude Code sends `custom: {defer_loading: true}` on tool definitions,
118
+ # which causes Bedrock to reject the request with "Extra inputs are not permitted"
119
+ # Ref: https://github.com/BerriAI/litellm/issues/22847
120
+ remove_custom_field_from_tools(_anthropic_request)
121
+
122
+ tools = optional_params.get("tools")
123
+ tool_search_used = self.is_tool_search_used(tools)
124
+ programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools)
125
+ input_examples_used = self.is_input_examples_used(tools)
126
+
127
+ beta_set = set(get_anthropic_beta_from_headers(headers))
128
+ auto_betas = self.get_anthropic_beta_list(
129
+ model=model,
130
+ optional_params=optional_params,
131
+ computer_tool_used=self.is_computer_tool_used(tools),
132
+ prompt_caching_set=False,
133
+ file_id_used=self.is_file_id_used(messages),
134
+ mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
135
+ )
136
+ beta_set.update(auto_betas)
137
+
138
+ if tool_search_used and not (
139
+ programmatic_tool_calling_used or input_examples_used
140
+ ):
141
+ beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
142
+ if "opus-4" in model.lower() or "opus_4" in model.lower():
143
+ beta_set.add("tool-search-tool-2025-10-19")
144
+
145
+ # Filter out beta headers that Bedrock Invoke doesn't support
146
+ # Uses centralized configuration from anthropic_beta_headers_config.json
147
+ beta_list = list(beta_set)
148
+ _anthropic_request["anthropic_beta"] = beta_list
149
+
150
+ return _anthropic_request
151
+
152
+ def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict:
153
+ """
154
+ Convert tool search entries to the format supported by the Bedrock Invoke API.
155
+ """
156
+ tools = optional_params.get("tools")
157
+ if not tools or not isinstance(tools, list):
158
+ return optional_params
159
+
160
+ normalized_tools = []
161
+ for tool in tools:
162
+ tool_type = tool.get("type")
163
+ if tool_type == "tool_search_tool_bm25_20251119":
164
+ # Bedrock Invoke does not support the BM25 variant, so skip it.
165
+ continue
166
+ if tool_type == "tool_search_tool_regex_20251119":
167
+ normalized_tool = tool.copy()
168
+ normalized_tool["type"] = "tool_search_tool_regex"
169
+ normalized_tool["name"] = normalized_tool.get(
170
+ "name", "tool_search_tool_regex"
171
+ )
172
+ normalized_tools.append(normalized_tool)
173
+ continue
174
+ normalized_tools.append(tool)
175
+
176
+ optional_params["tools"] = normalized_tools
177
+ return optional_params
178
+
179
+ def transform_response(
180
+ self,
181
+ model: str,
182
+ raw_response: httpx.Response,
183
+ model_response: ModelResponse,
184
+ logging_obj: LiteLLMLoggingObj,
185
+ request_data: dict,
186
+ messages: List[AllMessageValues],
187
+ optional_params: dict,
188
+ litellm_params: dict,
189
+ encoding: Any,
190
+ api_key: Optional[str] = None,
191
+ json_mode: Optional[bool] = None,
192
+ ) -> ModelResponse:
193
+ return AnthropicConfig.transform_response(
194
+ self,
195
+ model=model,
196
+ raw_response=raw_response,
197
+ model_response=model_response,
198
+ logging_obj=logging_obj,
199
+ request_data=request_data,
200
+ messages=messages,
201
+ optional_params=optional_params,
202
+ litellm_params=litellm_params,
203
+ encoding=encoding,
204
+ api_key=api_key,
205
+ json_mode=json_mode,
206
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py ADDED
@@ -0,0 +1,613 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import json
3
+ import time
4
+ from functools import partial
5
+ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast, get_args
6
+
7
+ import httpx
8
+
9
+ import litellm
10
+ from litellm._logging import verbose_logger
11
+ from litellm.litellm_core_utils.core_helpers import map_finish_reason
12
+ from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
13
+ from litellm.litellm_core_utils.prompt_templates.factory import (
14
+ cohere_message_pt,
15
+ custom_prompt,
16
+ deepseek_r1_pt,
17
+ prompt_factory,
18
+ )
19
+ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
20
+ from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call
21
+ from litellm.llms.bedrock.common_utils import BedrockError
22
+ from litellm.llms.custom_httpx.http_handler import (
23
+ AsyncHTTPHandler,
24
+ HTTPHandler,
25
+ _get_httpx_client,
26
+ )
27
+ from litellm.types.llms.openai import AllMessageValues
28
+ from litellm.types.utils import ModelResponse, Usage
29
+ from litellm.utils import CustomStreamWrapper
30
+
31
+ if TYPE_CHECKING:
32
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
33
+
34
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
35
+ else:
36
+ LiteLLMLoggingObj = Any
37
+
38
+ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
39
+
40
+
41
+ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
42
+ def __init__(self, **kwargs):
43
+ BaseConfig.__init__(self, **kwargs)
44
+ BaseAWSLLM.__init__(self, **kwargs)
45
+
46
+ def get_supported_openai_params(self, model: str) -> List[str]:
47
+ """
48
+ This is a base invoke model mapping. For Invoke - define a bedrock provider specific config that extends this class.
49
+ """
50
+ return [
51
+ "max_tokens",
52
+ "max_completion_tokens",
53
+ "stream",
54
+ ]
55
+
56
+ def map_openai_params(
57
+ self,
58
+ non_default_params: dict,
59
+ optional_params: dict,
60
+ model: str,
61
+ drop_params: bool,
62
+ ) -> dict:
63
+ """
64
+ This is a base invoke model mapping. For Invoke - define a bedrock provider specific config that extends this class.
65
+ """
66
+ for param, value in non_default_params.items():
67
+ if param == "max_tokens" or param == "max_completion_tokens":
68
+ optional_params["max_tokens"] = value
69
+ if param == "stream":
70
+ optional_params["stream"] = value
71
+ return optional_params
72
+
73
+ def get_complete_url(
74
+ self,
75
+ api_base: Optional[str],
76
+ api_key: Optional[str],
77
+ model: str,
78
+ optional_params: dict,
79
+ litellm_params: dict,
80
+ stream: Optional[bool] = None,
81
+ ) -> str:
82
+ """
83
+ Get the complete url for the request
84
+ """
85
+ provider = self.get_bedrock_invoke_provider(model)
86
+ modelId = self.get_bedrock_model_id(
87
+ model=model,
88
+ provider=provider,
89
+ optional_params=optional_params,
90
+ )
91
+ ### SET RUNTIME ENDPOINT ###
92
+ aws_bedrock_runtime_endpoint = optional_params.get(
93
+ "aws_bedrock_runtime_endpoint", None
94
+ ) # https://bedrock-runtime.{region_name}.amazonaws.com
95
+ endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
96
+ api_base=api_base,
97
+ aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
98
+ aws_region_name=self._get_aws_region_name(
99
+ optional_params=optional_params, model=model
100
+ ),
101
+ )
102
+
103
+ if (stream is not None and stream is True) and provider != "ai21":
104
+ endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream"
105
+ proxy_endpoint_url = (
106
+ f"{proxy_endpoint_url}/model/{modelId}/invoke-with-response-stream"
107
+ )
108
+ else:
109
+ endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
110
+ proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
111
+
112
+ return endpoint_url
113
+
114
+ def sign_request(
115
+ self,
116
+ headers: dict,
117
+ optional_params: dict,
118
+ request_data: dict,
119
+ api_base: str,
120
+ api_key: Optional[str] = None,
121
+ model: Optional[str] = None,
122
+ stream: Optional[bool] = None,
123
+ fake_stream: Optional[bool] = None,
124
+ ) -> Tuple[dict, Optional[bytes]]:
125
+ return self._sign_request(
126
+ service_name="bedrock",
127
+ headers=headers,
128
+ optional_params=optional_params,
129
+ request_data=request_data,
130
+ api_base=api_base,
131
+ api_key=api_key,
132
+ model=model,
133
+ stream=stream,
134
+ fake_stream=fake_stream,
135
+ )
136
+
137
+ def _apply_config_to_params(self, config: dict, inference_params: dict) -> None:
138
+ """Apply config values to inference_params if not already set."""
139
+ for k, v in config.items():
140
+ if k not in inference_params:
141
+ inference_params[k] = v
142
+
143
+ def transform_request(
144
+ self,
145
+ model: str,
146
+ messages: List[AllMessageValues],
147
+ optional_params: dict,
148
+ litellm_params: dict,
149
+ headers: dict,
150
+ ) -> dict:
151
+ ## SETUP ##
152
+ stream = optional_params.pop("stream", None)
153
+ custom_prompt_dict: dict = litellm_params.pop("custom_prompt_dict", None) or {}
154
+ hf_model_name = litellm_params.get("hf_model_name", None)
155
+
156
+ provider = self.get_bedrock_invoke_provider(model)
157
+
158
+ prompt, chat_history = self.convert_messages_to_prompt(
159
+ model=hf_model_name or model,
160
+ messages=messages,
161
+ provider=provider,
162
+ custom_prompt_dict=custom_prompt_dict,
163
+ )
164
+ inference_params = copy.deepcopy(optional_params)
165
+ inference_params = {
166
+ k: v
167
+ for k, v in inference_params.items()
168
+ if k not in self.aws_authentication_params
169
+ }
170
+ request_data: dict = {}
171
+ if provider == "cohere":
172
+ if model.startswith("cohere.command-r"):
173
+ ## LOAD CONFIG
174
+ config = litellm.AmazonCohereChatConfig().get_config()
175
+ self._apply_config_to_params(config, inference_params)
176
+ _data = {"message": prompt, **inference_params}
177
+ if chat_history is not None:
178
+ _data["chat_history"] = chat_history
179
+ request_data = _data
180
+ else:
181
+ ## LOAD CONFIG
182
+ config = litellm.AmazonCohereConfig.get_config()
183
+ self._apply_config_to_params(config, inference_params)
184
+ if stream is True:
185
+ inference_params[
186
+ "stream"
187
+ ] = True # cohere requires stream = True in inference params
188
+ request_data = {"prompt": prompt, **inference_params}
189
+ elif provider == "anthropic":
190
+ transformed_request = (
191
+ litellm.AmazonAnthropicClaudeConfig().transform_request(
192
+ model=model,
193
+ messages=messages,
194
+ optional_params=optional_params,
195
+ litellm_params=litellm_params,
196
+ headers=headers,
197
+ )
198
+ )
199
+
200
+ return transformed_request
201
+ elif provider == "nova":
202
+ return litellm.AmazonInvokeNovaConfig().transform_request(
203
+ model=model,
204
+ messages=messages,
205
+ optional_params=optional_params,
206
+ litellm_params=litellm_params,
207
+ headers=headers,
208
+ )
209
+ elif provider == "ai21":
210
+ ## LOAD CONFIG
211
+ config = litellm.AmazonAI21Config.get_config()
212
+ self._apply_config_to_params(config, inference_params)
213
+ request_data = {"prompt": prompt, **inference_params}
214
+ elif provider == "mistral":
215
+ ## LOAD CONFIG
216
+ config = litellm.AmazonMistralConfig.get_config()
217
+ self._apply_config_to_params(config, inference_params)
218
+ request_data = {"prompt": prompt, **inference_params}
219
+ elif provider == "amazon": # amazon titan
220
+ ## LOAD CONFIG
221
+ config = litellm.AmazonTitanConfig.get_config()
222
+ self._apply_config_to_params(config, inference_params)
223
+ request_data = {
224
+ "inputText": prompt,
225
+ "textGenerationConfig": inference_params,
226
+ }
227
+ elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
228
+ ## LOAD CONFIG
229
+ config = litellm.AmazonLlamaConfig.get_config()
230
+ self._apply_config_to_params(config, inference_params)
231
+ request_data = {"prompt": prompt, **inference_params}
232
+ elif provider == "twelvelabs":
233
+ return litellm.AmazonTwelveLabsPegasusConfig().transform_request(
234
+ model=model,
235
+ messages=messages,
236
+ optional_params=optional_params,
237
+ litellm_params=litellm_params,
238
+ headers=headers,
239
+ )
240
+ elif provider == "openai":
241
+ # OpenAI imported models use OpenAI Chat Completions format
242
+ return litellm.AmazonBedrockOpenAIConfig().transform_request(
243
+ model=model,
244
+ messages=messages,
245
+ optional_params=optional_params,
246
+ litellm_params=litellm_params,
247
+ headers=headers,
248
+ )
249
+ else:
250
+ raise BedrockError(
251
+ status_code=404,
252
+ message="Bedrock Invoke HTTPX: Unknown provider={}, model={}. Try calling via converse route - `bedrock/converse/<model>`.".format(
253
+ provider, model
254
+ ),
255
+ )
256
+
257
+ return request_data
258
+
259
+ def transform_response( # noqa: PLR0915
260
+ self,
261
+ model: str,
262
+ raw_response: httpx.Response,
263
+ model_response: ModelResponse,
264
+ logging_obj: LiteLLMLoggingObj,
265
+ request_data: dict,
266
+ messages: List[AllMessageValues],
267
+ optional_params: dict,
268
+ litellm_params: dict,
269
+ encoding: Any,
270
+ api_key: Optional[str] = None,
271
+ json_mode: Optional[bool] = None,
272
+ ) -> ModelResponse:
273
+ try:
274
+ completion_response = raw_response.json()
275
+ except Exception:
276
+ raise BedrockError(
277
+ message=raw_response.text, status_code=raw_response.status_code
278
+ )
279
+ verbose_logger.debug(
280
+ "bedrock invoke response % s",
281
+ json.dumps(completion_response, indent=4, default=str),
282
+ )
283
+ provider = self.get_bedrock_invoke_provider(model)
284
+ outputText: Optional[str] = None
285
+ try:
286
+ if provider == "cohere":
287
+ if "text" in completion_response:
288
+ outputText = completion_response["text"] # type: ignore
289
+ elif "generations" in completion_response:
290
+ outputText = completion_response["generations"][0]["text"]
291
+ model_response.choices[0].finish_reason = map_finish_reason(
292
+ completion_response["generations"][0]["finish_reason"]
293
+ )
294
+ elif provider == "anthropic":
295
+ return litellm.AmazonAnthropicClaudeConfig().transform_response(
296
+ model=model,
297
+ raw_response=raw_response,
298
+ model_response=model_response,
299
+ logging_obj=logging_obj,
300
+ request_data=request_data,
301
+ messages=messages,
302
+ optional_params=optional_params,
303
+ litellm_params=litellm_params,
304
+ encoding=encoding,
305
+ api_key=api_key,
306
+ json_mode=json_mode,
307
+ )
308
+ elif provider == "nova":
309
+ return litellm.AmazonInvokeNovaConfig().transform_response(
310
+ model=model,
311
+ raw_response=raw_response,
312
+ model_response=model_response,
313
+ logging_obj=logging_obj,
314
+ request_data=request_data,
315
+ messages=messages,
316
+ optional_params=optional_params,
317
+ litellm_params=litellm_params,
318
+ encoding=encoding,
319
+ )
320
+ elif provider == "twelvelabs":
321
+ return litellm.AmazonTwelveLabsPegasusConfig().transform_response(
322
+ model=model,
323
+ raw_response=raw_response,
324
+ model_response=model_response,
325
+ logging_obj=logging_obj,
326
+ request_data=request_data,
327
+ messages=messages,
328
+ optional_params=optional_params,
329
+ litellm_params=litellm_params,
330
+ encoding=encoding,
331
+ api_key=api_key,
332
+ json_mode=json_mode,
333
+ )
334
+ elif provider == "ai21":
335
+ outputText = (
336
+ completion_response.get("completions")[0].get("data").get("text")
337
+ )
338
+ elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
339
+ outputText = completion_response["generation"]
340
+ elif provider == "mistral":
341
+ outputText = litellm.AmazonMistralConfig.get_outputText(
342
+ completion_response, model_response
343
+ )
344
+ else: # amazon titan
345
+ outputText = completion_response.get("results")[0].get("outputText")
346
+ except Exception as e:
347
+ raise BedrockError(
348
+ message="Error processing={}, Received error={}".format(
349
+ raw_response.text, str(e)
350
+ ),
351
+ status_code=422,
352
+ )
353
+
354
+ try:
355
+ if (
356
+ outputText is not None
357
+ and len(outputText) > 0
358
+ and hasattr(model_response.choices[0], "message")
359
+ and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore
360
+ is None
361
+ ):
362
+ model_response.choices[0].message.content = outputText # type: ignore
363
+ elif (
364
+ hasattr(model_response.choices[0], "message")
365
+ and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore
366
+ is not None
367
+ ):
368
+ pass
369
+ else:
370
+ raise Exception()
371
+ except Exception as e:
372
+ raise BedrockError(
373
+ message="Error parsing received text={}.\nError-{}".format(
374
+ outputText, str(e)
375
+ ),
376
+ status_code=raw_response.status_code,
377
+ )
378
+
379
+ ## CALCULATING USAGE - bedrock returns usage in the headers
380
+ bedrock_input_tokens = raw_response.headers.get(
381
+ "x-amzn-bedrock-input-token-count", None
382
+ )
383
+ bedrock_output_tokens = raw_response.headers.get(
384
+ "x-amzn-bedrock-output-token-count", None
385
+ )
386
+
387
+ prompt_tokens = int(
388
+ bedrock_input_tokens or litellm.token_counter(messages=messages)
389
+ )
390
+
391
+ completion_tokens = int(
392
+ bedrock_output_tokens
393
+ or litellm.token_counter(
394
+ text=model_response.choices[0].message.content, # type: ignore
395
+ count_response_tokens=True,
396
+ )
397
+ )
398
+
399
+ model_response.created = int(time.time())
400
+ model_response.model = model
401
+ usage = Usage(
402
+ prompt_tokens=prompt_tokens,
403
+ completion_tokens=completion_tokens,
404
+ total_tokens=prompt_tokens + completion_tokens,
405
+ )
406
+ setattr(model_response, "usage", usage)
407
+
408
+ return model_response
409
+
410
+ def validate_environment(
411
+ self,
412
+ headers: dict,
413
+ model: str,
414
+ messages: List[AllMessageValues],
415
+ optional_params: dict,
416
+ litellm_params: dict,
417
+ api_key: Optional[str] = None,
418
+ api_base: Optional[str] = None,
419
+ ) -> dict:
420
+ return headers
421
+
422
+ def get_error_class(
423
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
424
+ ) -> BaseLLMException:
425
+ return BedrockError(status_code=status_code, message=error_message)
426
+
427
+ @track_llm_api_timing()
428
+ async def get_async_custom_stream_wrapper(
429
+ self,
430
+ model: str,
431
+ custom_llm_provider: str,
432
+ logging_obj: LiteLLMLoggingObj,
433
+ api_base: str,
434
+ headers: dict,
435
+ data: dict,
436
+ messages: list,
437
+ client: Optional[AsyncHTTPHandler] = None,
438
+ json_mode: Optional[bool] = None,
439
+ signed_json_body: Optional[bytes] = None,
440
+ ) -> CustomStreamWrapper:
441
+ streaming_response = CustomStreamWrapper(
442
+ completion_stream=None,
443
+ make_call=partial(
444
+ make_call,
445
+ client=client,
446
+ api_base=api_base,
447
+ headers=headers,
448
+ data=json.dumps(data),
449
+ model=model,
450
+ messages=messages,
451
+ logging_obj=logging_obj,
452
+ fake_stream=True if "ai21" in api_base else False,
453
+ bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
454
+ json_mode=json_mode,
455
+ ),
456
+ model=model,
457
+ custom_llm_provider="bedrock",
458
+ logging_obj=logging_obj,
459
+ )
460
+ return streaming_response
461
+
462
+ @track_llm_api_timing()
463
+ def get_sync_custom_stream_wrapper(
464
+ self,
465
+ model: str,
466
+ custom_llm_provider: str,
467
+ logging_obj: LiteLLMLoggingObj,
468
+ api_base: str,
469
+ headers: dict,
470
+ data: dict,
471
+ messages: list,
472
+ client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
473
+ json_mode: Optional[bool] = None,
474
+ signed_json_body: Optional[bytes] = None,
475
+ ) -> CustomStreamWrapper:
476
+ if client is None or isinstance(client, AsyncHTTPHandler):
477
+ client = _get_httpx_client(params={})
478
+ streaming_response = CustomStreamWrapper(
479
+ completion_stream=None,
480
+ make_call=partial(
481
+ make_sync_call,
482
+ client=client,
483
+ api_base=api_base,
484
+ headers=headers,
485
+ data=json.dumps(data),
486
+ signed_json_body=signed_json_body,
487
+ model=model,
488
+ messages=messages,
489
+ logging_obj=logging_obj,
490
+ fake_stream=True if "ai21" in api_base else False,
491
+ bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
492
+ json_mode=json_mode,
493
+ ),
494
+ model=model,
495
+ custom_llm_provider="bedrock",
496
+ logging_obj=logging_obj,
497
+ )
498
+ return streaming_response
499
+
500
+ @property
501
+ def has_custom_stream_wrapper(self) -> bool:
502
+ return True
503
+
504
+ @property
505
+ def supports_stream_param_in_request_body(self) -> bool:
506
+ """
507
+ Bedrock invoke does not allow passing `stream` in the request body.
508
+ """
509
+ return False
510
+
511
+ @staticmethod
512
+ def get_bedrock_invoke_provider(
513
+ model: str,
514
+ ) -> Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL]:
515
+ """
516
+ Helper function to get the bedrock provider from the model
517
+
518
+ handles 4 scenarios:
519
+ 1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
520
+ 2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
521
+ 3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
522
+ 4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
523
+ """
524
+ if model.startswith("invoke/"):
525
+ model = model.replace("invoke/", "", 1)
526
+
527
+ # Special case: Check for "nova" in model name first (before "amazon")
528
+ # This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
529
+ if "nova" in model.lower():
530
+ if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
531
+ return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
532
+
533
+ _split_model = model.split(".")[0]
534
+ if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
535
+ return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
536
+
537
+ # If not a known provider, check for pattern with two slashes
538
+ provider = AmazonInvokeConfig._get_provider_from_model_path(model)
539
+ if provider is not None:
540
+ return provider
541
+
542
+ for provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
543
+ if provider in model:
544
+ return provider
545
+ return None
546
+
547
+ @staticmethod
548
+ def _get_provider_from_model_path(
549
+ model_path: str,
550
+ ) -> Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL]:
551
+ """
552
+ Helper function to get the provider from a model path with format: provider/model-name
553
+
554
+ Args:
555
+ model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
556
+
557
+ Returns:
558
+ Optional[str]: The provider name, or None if no valid provider found
559
+ """
560
+ parts = model_path.split("/")
561
+ if len(parts) >= 1:
562
+ provider = parts[0]
563
+ if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
564
+ return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
565
+ return None
566
+
567
+ def convert_messages_to_prompt(
568
+ self, model, messages, provider, custom_prompt_dict
569
+ ) -> Tuple[str, Optional[list]]:
570
+ # handle anthropic prompts and amazon titan prompts
571
+ prompt = ""
572
+ chat_history: Optional[list] = None
573
+ ## CUSTOM PROMPT
574
+ if model in custom_prompt_dict:
575
+ # check if the model has a registered custom prompt
576
+ model_prompt_details = custom_prompt_dict[model]
577
+ prompt = custom_prompt(
578
+ role_dict=model_prompt_details["roles"],
579
+ initial_prompt_value=model_prompt_details.get(
580
+ "initial_prompt_value", ""
581
+ ),
582
+ final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
583
+ messages=messages,
584
+ )
585
+ return prompt, None
586
+ ## ELSE
587
+ if provider == "anthropic" or provider == "amazon":
588
+ prompt = prompt_factory(
589
+ model=model, messages=messages, custom_llm_provider="bedrock"
590
+ )
591
+ elif provider == "mistral":
592
+ prompt = prompt_factory(
593
+ model=model, messages=messages, custom_llm_provider="bedrock"
594
+ )
595
+ elif provider == "meta" or provider == "llama":
596
+ prompt = prompt_factory(
597
+ model=model, messages=messages, custom_llm_provider="bedrock"
598
+ )
599
+ elif provider == "cohere":
600
+ prompt, chat_history = cohere_message_pt(messages=messages)
601
+ elif provider == "deepseek_r1":
602
+ prompt = deepseek_r1_pt(messages=messages)
603
+ else:
604
+ prompt = ""
605
+ for message in messages:
606
+ if "role" in message:
607
+ if message["role"] == "user":
608
+ prompt += f"{message['content']}"
609
+ else:
610
+ prompt += f"{message['content']}"
611
+ else:
612
+ prompt += f"{message['content']}"
613
+ return prompt, chat_history # type: ignore
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/common_utils.py ADDED
@@ -0,0 +1,1140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ """
4
+ Common utilities used across bedrock chat/embedding/image generation
5
+ """
6
+
7
+ import json
8
+ import os
9
+ from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Union
10
+
11
+ if TYPE_CHECKING:
12
+ from litellm.types.llms.bedrock import BedrockCreateBatchRequest
13
+
14
+ import httpx
15
+
16
+ import litellm
17
+ from litellm.llms.base_llm.anthropic_messages.transformation import (
18
+ BaseAnthropicMessagesConfig,
19
+ )
20
+ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
21
+ from litellm.llms.base_llm.chat.transformation import BaseLLMException
22
+ from litellm.secret_managers.main import get_secret
23
+
24
+ if TYPE_CHECKING:
25
+ from litellm.types.llms.openai import AllMessageValues
26
+
27
+
28
+ class BedrockError(BaseLLMException):
29
+ pass
30
+
31
+
32
+ # Lazy import cache to avoid circular imports and performance impact
33
+ _get_model_info = None
34
+
35
+
36
+ def get_cached_model_info():
37
+ """
38
+ Lazy import and cache get_model_info to avoid circular imports.
39
+
40
+ This function is used by bedrock transformation classes that need get_model_info
41
+ but cannot import it at module level due to circular import issues.
42
+ The function is cached after first use to avoid performance impact.
43
+ """
44
+ global _get_model_info
45
+ if _get_model_info is None:
46
+ from litellm import get_model_info
47
+
48
+ _get_model_info = get_model_info
49
+ return _get_model_info
50
+
51
+
52
+ def remove_custom_field_from_tools(request_body: dict) -> None:
53
+ """
54
+ Remove ``custom`` field from each tool in the request body.
55
+
56
+ Claude Code (v2.1.69+) sends ``custom: {defer_loading: true}`` on tool
57
+ definitions, which Anthropic's API accepts but Bedrock rejects with
58
+ ``"Extra inputs are not permitted"``.
59
+
60
+ Args:
61
+ request_body: The request dictionary to modify in-place.
62
+
63
+ Ref: https://github.com/BerriAI/litellm/issues/22847
64
+ """
65
+ tools = request_body.get("tools")
66
+ if not tools or not isinstance(tools, list):
67
+ return
68
+ for tool in tools:
69
+ if isinstance(tool, dict):
70
+ tool.pop("custom", None)
71
+
72
+
73
+ class AmazonBedrockGlobalConfig:
74
+ def __init__(self):
75
+ pass
76
+
77
+ def get_mapped_special_auth_params(self) -> dict:
78
+ """
79
+ Mapping of common auth params across bedrock/vertex/azure/watsonx
80
+ """
81
+ return {"region_name": "aws_region_name"}
82
+
83
+ def map_special_auth_params(self, non_default_params: dict, optional_params: dict):
84
+ mapped_params = self.get_mapped_special_auth_params()
85
+ for param, value in non_default_params.items():
86
+ if param in mapped_params:
87
+ optional_params[mapped_params[param]] = value
88
+ return optional_params
89
+
90
+ def get_all_regions(self) -> List[str]:
91
+ return (
92
+ self.get_us_regions()
93
+ + self.get_eu_regions()
94
+ + self.get_ap_regions()
95
+ + self.get_ca_regions()
96
+ + self.get_sa_regions()
97
+ )
98
+
99
+ def get_ap_regions(self) -> List[str]:
100
+ """
101
+ Source: https://www.aws-services.info/bedrock.html
102
+ """
103
+ return [
104
+ "ap-northeast-1", # Asia Pacific (Tokyo)
105
+ "ap-northeast-2", # Asia Pacific (Seoul)
106
+ "ap-northeast-3", # Asia Pacific (Osaka)
107
+ "ap-south-1", # Asia Pacific (Mumbai)
108
+ "ap-south-2", # Asia Pacific (Hyderabad)
109
+ "ap-southeast-1", # Asia Pacific (Singapore)
110
+ "ap-southeast-2", # Asia Pacific (Sydney)
111
+ ]
112
+
113
+ def get_sa_regions(self) -> List[str]:
114
+ return ["sa-east-1"]
115
+
116
+ def get_eu_regions(self) -> List[str]:
117
+ """
118
+ Source: https://www.aws-services.info/bedrock.html
119
+ """
120
+ return [
121
+ "eu-west-1", # Europe (Ireland)
122
+ "eu-west-2", # Europe (London)
123
+ "eu-west-3", # Europe (Paris)
124
+ "eu-central-1", # Europe (Frankfurt)
125
+ "eu-central-2", # Europe (Zurich)
126
+ "eu-south-1", # Europe (Milan)
127
+ "eu-south-2", # Europe (Spain)
128
+ "eu-north-1", # Europe (Stockholm)
129
+ ]
130
+
131
+ def get_ca_regions(self) -> List[str]:
132
+ return ["ca-central-1"]
133
+
134
+ def get_us_regions(self) -> List[str]:
135
+ """
136
+ Source: https://www.aws-services.info/bedrock.html
137
+ """
138
+ return [
139
+ "us-east-1", # US East (N. Virginia)
140
+ "us-east-2", # US East (Ohio)
141
+ "us-west-1", # US West (N. California)
142
+ "us-west-2", # US West (Oregon)
143
+ "us-gov-east-1", # AWS GovCloud (US-East)
144
+ "us-gov-west-1", # AWS GovCloud (US-West)
145
+ ]
146
+
147
+
148
+ def add_custom_header(headers):
149
+ """Closure to capture the headers and add them."""
150
+
151
+ def callback(request, **kwargs):
152
+ """Actual callback function that Boto3 will call."""
153
+ for header_name, header_value in headers.items():
154
+ request.headers.add_header(header_name, header_value)
155
+
156
+ return callback
157
+
158
+
159
+ def _get_bedrock_client_ssl_verify() -> Union[bool, str]:
160
+ """
161
+ Get SSL verification setting for Bedrock client.
162
+
163
+ Returns the SSL verification setting which can be:
164
+ - True: Use default SSL verification
165
+ - False: Disable SSL verification
166
+ - str: Path to a custom CA bundle file
167
+ """
168
+ from litellm.llms.custom_httpx.http_handler import get_ssl_verify
169
+
170
+ return get_ssl_verify()
171
+
172
+
173
+ def init_bedrock_client(
174
+ region_name=None,
175
+ aws_access_key_id: Optional[str] = None,
176
+ aws_secret_access_key: Optional[str] = None,
177
+ aws_region_name: Optional[str] = None,
178
+ aws_bedrock_runtime_endpoint: Optional[str] = None,
179
+ aws_session_name: Optional[str] = None,
180
+ aws_profile_name: Optional[str] = None,
181
+ aws_role_name: Optional[str] = None,
182
+ aws_web_identity_token: Optional[str] = None,
183
+ extra_headers: Optional[dict] = None,
184
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
185
+ ):
186
+ # check for custom AWS_REGION_NAME and use it if not passed to init_bedrock_client
187
+ litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
188
+ standard_aws_region_name = get_secret("AWS_REGION", None)
189
+ ## CHECK IS 'os.environ/' passed in
190
+ # Define the list of parameters to check
191
+ params_to_check = [
192
+ aws_access_key_id,
193
+ aws_secret_access_key,
194
+ aws_region_name,
195
+ aws_bedrock_runtime_endpoint,
196
+ aws_session_name,
197
+ aws_profile_name,
198
+ aws_role_name,
199
+ aws_web_identity_token,
200
+ ]
201
+
202
+ # Iterate over parameters and update if needed
203
+ for i, param in enumerate(params_to_check):
204
+ if param and param.startswith("os.environ/"):
205
+ params_to_check[i] = get_secret(param) # type: ignore
206
+ # Assign updated values back to parameters
207
+ (
208
+ aws_access_key_id,
209
+ aws_secret_access_key,
210
+ aws_region_name,
211
+ aws_bedrock_runtime_endpoint,
212
+ aws_session_name,
213
+ aws_profile_name,
214
+ aws_role_name,
215
+ aws_web_identity_token,
216
+ ) = params_to_check
217
+
218
+ ssl_verify = _get_bedrock_client_ssl_verify()
219
+
220
+ ### SET REGION NAME
221
+ if region_name:
222
+ pass
223
+ elif aws_region_name:
224
+ region_name = aws_region_name
225
+ elif litellm_aws_region_name:
226
+ region_name = litellm_aws_region_name
227
+ elif standard_aws_region_name:
228
+ region_name = standard_aws_region_name
229
+ else:
230
+ raise BedrockError(
231
+ message="AWS region not set: set AWS_REGION_NAME or AWS_REGION env variable or in .env file",
232
+ status_code=401,
233
+ )
234
+
235
+ # check for custom AWS_BEDROCK_RUNTIME_ENDPOINT and use it if not passed to init_bedrock_client
236
+ env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT")
237
+ if aws_bedrock_runtime_endpoint:
238
+ endpoint_url = aws_bedrock_runtime_endpoint
239
+ elif env_aws_bedrock_runtime_endpoint:
240
+ endpoint_url = env_aws_bedrock_runtime_endpoint
241
+ else:
242
+ endpoint_url = f"https://bedrock-runtime.{region_name}.amazonaws.com"
243
+
244
+ import boto3
245
+
246
+ if isinstance(timeout, float):
247
+ config = boto3.session.Config(connect_timeout=timeout, read_timeout=timeout) # type: ignore
248
+ elif isinstance(timeout, httpx.Timeout):
249
+ config = boto3.session.Config( # type: ignore
250
+ connect_timeout=timeout.connect, read_timeout=timeout.read
251
+ )
252
+ else:
253
+ config = boto3.session.Config() # type: ignore
254
+
255
+ ### CHECK STS ###
256
+ if (
257
+ aws_web_identity_token is not None
258
+ and aws_role_name is not None
259
+ and aws_session_name is not None
260
+ ):
261
+ oidc_token = get_secret(aws_web_identity_token)
262
+
263
+ if oidc_token is None:
264
+ raise BedrockError(
265
+ message="OIDC token could not be retrieved from secret manager.",
266
+ status_code=401,
267
+ )
268
+
269
+ sts_client = boto3.client("sts", verify=ssl_verify)
270
+
271
+ # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
272
+ # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
273
+ sts_response = sts_client.assume_role_with_web_identity(
274
+ RoleArn=aws_role_name,
275
+ RoleSessionName=aws_session_name,
276
+ WebIdentityToken=oidc_token,
277
+ DurationSeconds=3600,
278
+ )
279
+
280
+ client = boto3.client(
281
+ service_name="bedrock-runtime",
282
+ aws_access_key_id=sts_response["Credentials"]["AccessKeyId"],
283
+ aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"],
284
+ aws_session_token=sts_response["Credentials"]["SessionToken"],
285
+ region_name=region_name,
286
+ endpoint_url=endpoint_url,
287
+ config=config,
288
+ verify=ssl_verify,
289
+ )
290
+ elif aws_role_name is not None and aws_session_name is not None:
291
+ # use sts if role name passed in
292
+ sts_client = boto3.client(
293
+ "sts",
294
+ aws_access_key_id=aws_access_key_id,
295
+ aws_secret_access_key=aws_secret_access_key,
296
+ verify=ssl_verify,
297
+ )
298
+
299
+ sts_response = sts_client.assume_role(
300
+ RoleArn=aws_role_name, RoleSessionName=aws_session_name
301
+ )
302
+
303
+ client = boto3.client(
304
+ service_name="bedrock-runtime",
305
+ aws_access_key_id=sts_response["Credentials"]["AccessKeyId"],
306
+ aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"],
307
+ aws_session_token=sts_response["Credentials"]["SessionToken"],
308
+ region_name=region_name,
309
+ endpoint_url=endpoint_url,
310
+ config=config,
311
+ verify=ssl_verify,
312
+ )
313
+ elif aws_access_key_id is not None:
314
+ # uses auth params passed to completion
315
+ # aws_access_key_id is not None, assume user is trying to auth using litellm.completion
316
+
317
+ client = boto3.client(
318
+ service_name="bedrock-runtime",
319
+ aws_access_key_id=aws_access_key_id,
320
+ aws_secret_access_key=aws_secret_access_key,
321
+ region_name=region_name,
322
+ endpoint_url=endpoint_url,
323
+ config=config,
324
+ verify=ssl_verify,
325
+ )
326
+ elif aws_profile_name is not None:
327
+ # uses auth values from AWS profile usually stored in ~/.aws/credentials
328
+
329
+ client = boto3.Session(profile_name=aws_profile_name).client(
330
+ service_name="bedrock-runtime",
331
+ region_name=region_name,
332
+ endpoint_url=endpoint_url,
333
+ config=config,
334
+ verify=ssl_verify,
335
+ )
336
+ else:
337
+ # aws_access_key_id is None, assume user is trying to auth using env variables
338
+ # boto3 automatically reads env variables
339
+
340
+ client = boto3.client(
341
+ service_name="bedrock-runtime",
342
+ region_name=region_name,
343
+ endpoint_url=endpoint_url,
344
+ config=config,
345
+ verify=ssl_verify,
346
+ )
347
+ if extra_headers:
348
+ client.meta.events.register(
349
+ "before-sign.bedrock-runtime.*", add_custom_header(extra_headers)
350
+ )
351
+
352
+ return client
353
+
354
+
355
+ class ModelResponseIterator:
356
+ def __init__(self, model_response):
357
+ self.model_response = model_response
358
+ self.is_done = False
359
+
360
+ # Sync iterator
361
+ def __iter__(self):
362
+ return self
363
+
364
+ def __next__(self):
365
+ if self.is_done:
366
+ raise StopIteration
367
+ self.is_done = True
368
+ return self.model_response
369
+
370
+ # Async iterator
371
+ def __aiter__(self):
372
+ return self
373
+
374
+ async def __anext__(self):
375
+ if self.is_done:
376
+ raise StopAsyncIteration
377
+ self.is_done = True
378
+ return self.model_response
379
+
380
+
381
+ def get_bedrock_tool_name(response_tool_name: str) -> str:
382
+ """
383
+ If litellm formatted the input tool name, we need to convert it back to the original name.
384
+
385
+ Args:
386
+ response_tool_name (str): The name of the tool as received from the response.
387
+
388
+ Returns:
389
+ str: The original name of the tool.
390
+ """
391
+
392
+ if response_tool_name in litellm.bedrock_tool_name_mappings.cache_dict:
393
+ response_tool_name = litellm.bedrock_tool_name_mappings.cache_dict[
394
+ response_tool_name
395
+ ]
396
+ return response_tool_name
397
+
398
+
399
+ # Cache the global regions list at module level
400
+ _BEDROCK_GLOBAL_REGIONS: Optional[List[str]] = None
401
+
402
+
403
+ def _get_all_bedrock_regions() -> List[str]:
404
+ """Get all Bedrock regions, cached at module level."""
405
+ global _BEDROCK_GLOBAL_REGIONS
406
+ if _BEDROCK_GLOBAL_REGIONS is None:
407
+ _BEDROCK_GLOBAL_REGIONS = AmazonBedrockGlobalConfig().get_all_regions()
408
+ return _BEDROCK_GLOBAL_REGIONS
409
+
410
+
411
+ def get_bedrock_cross_region_inference_regions() -> List[str]:
412
+ """Abbreviations of regions AWS Bedrock supports for cross region inference."""
413
+ return ["global", "us", "eu", "apac", "jp", "au", "us-gov"]
414
+
415
+
416
+ def extract_model_name_from_bedrock_arn(model: str) -> str:
417
+ """
418
+ Extract the model name from an AWS Bedrock ARN.
419
+ Returns the string after the last '/' if 'arn' is in the input string.
420
+ """
421
+ if "arn" in model.lower():
422
+ return model.split("/")[-1]
423
+ return model
424
+
425
+
426
+ def strip_bedrock_routing_prefix(model: str) -> str:
427
+ """Strip LiteLLM routing prefixes from model name."""
428
+ for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "nova-2/", "nova/"]:
429
+ if model.startswith(prefix):
430
+ model = model.split("/", 1)[1]
431
+ return model
432
+
433
+
434
+ def strip_bedrock_throughput_suffix(model: str) -> str:
435
+ """Strip throughput tier suffixes from Bedrock model names."""
436
+ import re
437
+
438
+ # Pattern matches model:version:throughput where throughput is like 51k, 18k, etc.
439
+ # Keep the model:version part, strip the :throughput suffix
440
+ return re.sub(r"(:\d+):\d+k$", r"\1", model)
441
+
442
+
443
+ def get_bedrock_base_model(model: str) -> str:
444
+ """
445
+ Get the base model from the given model name.
446
+
447
+ Handle model names like:
448
+ - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1"
449
+ - "bedrock/converse/model" -> "model"
450
+ - "anthropic.claude-3-5-sonnet-20241022-v2:0:51k" -> "anthropic.claude-3-5-sonnet-20241022-v2:0"
451
+ - "bedrock/nova-2/arn:aws:..." -> "amazon.nova-2-custom"
452
+ - "bedrock/nova/arn:aws:..." -> "amazon.nova-custom"
453
+ """
454
+ # Detect nova spec prefixes before stripping them
455
+ stripped = model
456
+ for rp in ["bedrock/converse/", "bedrock/", "converse/"]:
457
+ if stripped.startswith(rp):
458
+ stripped = stripped[len(rp) :]
459
+ break
460
+ if stripped.startswith("nova-2/"):
461
+ return "amazon.nova-2-custom"
462
+ elif stripped.startswith("nova/"):
463
+ return "amazon.nova-custom"
464
+
465
+ model = strip_bedrock_routing_prefix(model)
466
+ model = extract_model_name_from_bedrock_arn(model)
467
+ model = strip_bedrock_throughput_suffix(model)
468
+
469
+ potential_region = model.split(".", 1)[0]
470
+ alt_potential_region = model.split("/", 1)[0]
471
+
472
+ if potential_region in get_bedrock_cross_region_inference_regions():
473
+ return model.split(".", 1)[1]
474
+ elif (
475
+ alt_potential_region in _get_all_bedrock_regions()
476
+ and len(model.split("/", 1)) > 1
477
+ ):
478
+ return model.split("/", 1)[1]
479
+
480
+ return model
481
+
482
+
483
+ def is_claude_4_5_on_bedrock(model: str) -> bool:
484
+ """
485
+ Check if the model is a Claude 4.5 model on Bedrock.
486
+ Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
487
+ """
488
+ model_lower = model.lower()
489
+ claude_4_5_patterns = [
490
+ "sonnet-4.5",
491
+ "sonnet_4.5",
492
+ "sonnet-4-5",
493
+ "sonnet_4_5",
494
+ "haiku-4.5",
495
+ "haiku_4.5",
496
+ "haiku-4-5",
497
+ "haiku_4_5",
498
+ "opus-4.5",
499
+ "opus_4.5",
500
+ "opus-4-5",
501
+ "opus_4_5",
502
+ "sonnet-4.6",
503
+ "sonnet_4.6",
504
+ "sonnet-4-6",
505
+ "sonnet_4_6",
506
+ "opus-4.6",
507
+ "opus_4.6",
508
+ "opus-4-6",
509
+ "opus_4_6",
510
+ ]
511
+ return any(pattern in model_lower for pattern in claude_4_5_patterns)
512
+
513
+
514
+ # Import after standalone functions to avoid circular imports
515
+ from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
516
+
517
+
518
+ class BedrockModelInfo(BaseLLMModelInfo):
519
+ global_config = AmazonBedrockGlobalConfig()
520
+ all_global_regions = global_config.get_all_regions()
521
+
522
+ @staticmethod
523
+ def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
524
+ """
525
+ Get the API base for the given model.
526
+ """
527
+ return api_base
528
+
529
+ @staticmethod
530
+ def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
531
+ """
532
+ Get the API key for the given model.
533
+ """
534
+ return api_key
535
+
536
+ def validate_environment(
537
+ self,
538
+ headers: dict,
539
+ model: str,
540
+ messages: List["AllMessageValues"],
541
+ optional_params: dict,
542
+ litellm_params: dict,
543
+ api_key: Optional[str] = None,
544
+ api_base: Optional[str] = None,
545
+ ) -> dict:
546
+ return headers
547
+
548
+ def get_models(
549
+ self, api_key: Optional[str] = None, api_base: Optional[str] = None
550
+ ) -> List[str]:
551
+ return []
552
+
553
+ # def get_provider_info(self, model: str) -> Optional[ProviderSpecificModelInfo]:
554
+ # """
555
+ # Handles Bedrock throughput suffixes like ":28k", ":51k".
556
+ # """
557
+ # import re
558
+
559
+ # overrides: ProviderSpecificModelInfo = {}
560
+
561
+ # # Parse context window suffix (e.g., :28k, :51k)
562
+ # match = re.search(r":(\d+)k$", model)
563
+ # if match:
564
+ # throughput_value = int(match.group(1)) * 1000
565
+ # overrides["max_input_tokens"] = throughput_value
566
+
567
+ # return overrides if overrides else None
568
+
569
+ def get_token_counter(self) -> Optional[BaseTokenCounter]:
570
+ """
571
+ Factory method to create a Bedrock token counter.
572
+
573
+ Returns:
574
+ BedrockTokenCounter instance for this provider.
575
+ """
576
+ return BedrockTokenCounter()
577
+
578
+ @staticmethod
579
+ def extract_model_name_from_arn(model: str) -> str:
580
+ """Wrapper for standalone function. See extract_model_name_from_bedrock_arn()."""
581
+ return extract_model_name_from_bedrock_arn(model)
582
+
583
+ @staticmethod
584
+ def get_non_litellm_routing_model_name(model: str) -> str:
585
+ """Wrapper for standalone function. See strip_bedrock_routing_prefix()."""
586
+ return strip_bedrock_routing_prefix(model)
587
+
588
+ @staticmethod
589
+ def get_base_model(model: str) -> str:
590
+ """Wrapper for standalone function. See get_bedrock_base_model()."""
591
+ return get_bedrock_base_model(model)
592
+
593
+ @staticmethod
594
+ def _supported_cross_region_inference_region() -> List[str]:
595
+ """Wrapper for standalone function. See get_bedrock_cross_region_inference_regions()."""
596
+ return get_bedrock_cross_region_inference_regions()
597
+
598
+ @staticmethod
599
+ def get_bedrock_route(
600
+ model: str,
601
+ ) -> Literal[
602
+ "converse",
603
+ "invoke",
604
+ "converse_like",
605
+ "agent",
606
+ "agentcore",
607
+ "async_invoke",
608
+ "openai",
609
+ ]:
610
+ """
611
+ Get the bedrock route for the given model.
612
+ """
613
+ route_mappings: Dict[
614
+ str,
615
+ Literal[
616
+ "invoke",
617
+ "converse_like",
618
+ "converse",
619
+ "agent",
620
+ "agentcore",
621
+ "async_invoke",
622
+ "openai",
623
+ ],
624
+ ] = {
625
+ "invoke/": "invoke",
626
+ "converse_like/": "converse_like",
627
+ "converse/": "converse",
628
+ "agent/": "agent",
629
+ "agentcore/": "agentcore",
630
+ "async_invoke/": "async_invoke",
631
+ "openai/": "openai",
632
+ }
633
+
634
+ # Check explicit routes first
635
+ for prefix, route_type in route_mappings.items():
636
+ if prefix in model:
637
+ return route_type
638
+
639
+ # Check for nova spec prefixes (nova/ and nova-2/)
640
+ _model_after_bedrock = model.replace("bedrock/", "", 1)
641
+ if _model_after_bedrock.startswith(
642
+ "nova-2/"
643
+ ) or _model_after_bedrock.startswith("nova/"):
644
+ return "converse"
645
+
646
+ base_model = BedrockModelInfo.get_base_model(model)
647
+ alt_model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model)
648
+ if (
649
+ base_model in litellm.bedrock_converse_models
650
+ or alt_model in litellm.bedrock_converse_models
651
+ ):
652
+ return "converse"
653
+ return "invoke"
654
+
655
+ @staticmethod
656
+ def _explicit_converse_route(model: str) -> bool:
657
+ """
658
+ Check if the model is an explicit converse route.
659
+ """
660
+ return "converse/" in model
661
+
662
+ @staticmethod
663
+ def _explicit_invoke_route(model: str) -> bool:
664
+ """
665
+ Check if the model is an explicit invoke route.
666
+ """
667
+ return "invoke/" in model
668
+
669
+ @staticmethod
670
+ def _explicit_agent_route(model: str) -> bool:
671
+ """
672
+ Check if the model is an explicit agent route.
673
+ """
674
+ return "agent/" in model
675
+
676
+ @staticmethod
677
+ def _explicit_agentcore_route(model: str) -> bool:
678
+ """
679
+ Check if the model is an explicit agentcore route.
680
+ """
681
+ return "agentcore/" in model
682
+
683
+ @staticmethod
684
+ def _explicit_converse_like_route(model: str) -> bool:
685
+ """
686
+ Check if the model is an explicit converse like route.
687
+ """
688
+ return "converse_like/" in model
689
+
690
+ @staticmethod
691
+ def _explicit_async_invoke_route(model: str) -> bool:
692
+ """
693
+ Check if the model is an explicit async invoke route.
694
+ """
695
+ return "async_invoke/" in model
696
+
697
+ @staticmethod
698
+ def _explicit_openai_route(model: str) -> bool:
699
+ """
700
+ Check if the model is an explicit openai route.
701
+ Used for Bedrock imported models that use OpenAI Chat Completions format.
702
+ """
703
+ return "openai/" in model
704
+
705
+ @staticmethod
706
+ def get_bedrock_provider_config_for_messages_api(
707
+ model: str,
708
+ ) -> Optional[BaseAnthropicMessagesConfig]:
709
+ """
710
+ Get the bedrock provider config for the given model.
711
+
712
+ Only route to AmazonAnthropicClaude3MessagesConfig() for BaseMessagesConfig
713
+
714
+ All other routes should return None since they will go through litellm.completion
715
+ """
716
+
717
+ #########################################################
718
+ # Converse routes should go through litellm.completion()
719
+ if BedrockModelInfo._explicit_converse_route(model):
720
+ return None
721
+
722
+ #########################################################
723
+ # This goes through litellm.AmazonAnthropicClaude3MessagesConfig()
724
+ # Since bedrock Invoke supports Native Anthropic Messages API
725
+ #########################################################
726
+ if "claude" in model:
727
+ return litellm.AmazonAnthropicClaudeMessagesConfig()
728
+
729
+ #########################################################
730
+ # These routes will go through litellm.completion()
731
+ #########################################################
732
+ return None
733
+
734
+
735
+ def get_bedrock_chat_config(model: str):
736
+ """
737
+ Helper function to get the appropriate Bedrock chat config based on model and route.
738
+
739
+ Args:
740
+ model: The model name/identifier
741
+
742
+ Returns:
743
+ The appropriate Bedrock config class instance
744
+ """
745
+ bedrock_route = BedrockModelInfo.get_bedrock_route(model)
746
+ bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider(
747
+ model=model
748
+ )
749
+ base_model = BedrockModelInfo.get_base_model(model)
750
+
751
+ # Handle explicit routes first
752
+ if bedrock_route == "converse" or bedrock_route == "converse_like":
753
+ return litellm.AmazonConverseConfig()
754
+ elif bedrock_route == "openai":
755
+ return litellm.AmazonBedrockOpenAIConfig()
756
+ elif bedrock_route == "agent":
757
+ from litellm.llms.bedrock.chat.invoke_agent.transformation import (
758
+ AmazonInvokeAgentConfig,
759
+ )
760
+
761
+ return AmazonInvokeAgentConfig()
762
+ elif bedrock_route == "agentcore":
763
+ from litellm.llms.bedrock.chat.agentcore.transformation import (
764
+ AmazonAgentCoreConfig,
765
+ )
766
+
767
+ return AmazonAgentCoreConfig()
768
+
769
+ # Handle provider-specific configs
770
+ if bedrock_invoke_provider == "amazon":
771
+ return litellm.AmazonTitanConfig()
772
+ elif bedrock_invoke_provider == "anthropic":
773
+ if (
774
+ base_model
775
+ in litellm.AmazonAnthropicConfig.get_legacy_anthropic_model_names()
776
+ ):
777
+ return litellm.AmazonAnthropicConfig()
778
+ else:
779
+ return litellm.AmazonAnthropicClaudeConfig()
780
+ elif bedrock_invoke_provider == "meta" or bedrock_invoke_provider == "llama":
781
+ return litellm.AmazonLlamaConfig()
782
+ elif bedrock_invoke_provider == "ai21":
783
+ return litellm.AmazonAI21Config()
784
+ elif bedrock_invoke_provider == "cohere":
785
+ return litellm.AmazonCohereConfig()
786
+ elif bedrock_invoke_provider == "mistral":
787
+ return litellm.AmazonMistralConfig()
788
+ elif bedrock_invoke_provider == "moonshot":
789
+ return litellm.AmazonMoonshotConfig()
790
+ elif bedrock_invoke_provider == "deepseek_r1":
791
+ return litellm.AmazonDeepSeekR1Config()
792
+ elif bedrock_invoke_provider == "nova":
793
+ return litellm.AmazonInvokeNovaConfig()
794
+ elif bedrock_invoke_provider == "qwen3":
795
+ return litellm.AmazonQwen3Config()
796
+ elif bedrock_invoke_provider == "qwen2":
797
+ return litellm.AmazonQwen2Config()
798
+ elif bedrock_invoke_provider == "twelvelabs":
799
+ return litellm.AmazonTwelveLabsPegasusConfig()
800
+ else:
801
+ return litellm.AmazonInvokeConfig()
802
+
803
+
804
+ class BedrockEventStreamDecoderBase:
805
+ """
806
+ Base class for event stream decoding for Bedrock
807
+ """
808
+
809
+ _response_stream_shape_cache = None
810
+
811
+ def __init__(self):
812
+ from botocore.parsers import EventStreamJSONParser
813
+
814
+ self.parser = EventStreamJSONParser()
815
+
816
+ def get_response_stream_shape(self):
817
+ if self._response_stream_shape_cache is None:
818
+ from botocore.loaders import Loader
819
+ from botocore.model import ServiceModel
820
+
821
+ loader = Loader()
822
+ bedrock_service_dict = loader.load_service_model(
823
+ "bedrock-runtime", "service-2"
824
+ )
825
+ bedrock_service_model = ServiceModel(bedrock_service_dict)
826
+ self._response_stream_shape_cache = bedrock_service_model.shape_for(
827
+ "ResponseStream"
828
+ )
829
+
830
+ return self._response_stream_shape_cache
831
+
832
+ def _parse_message_from_event(self, event) -> Optional[str]:
833
+ response_dict = event.to_response_dict()
834
+ parsed_response = self.parser.parse(
835
+ response_dict, self.get_response_stream_shape()
836
+ )
837
+
838
+ if response_dict["status_code"] != 200:
839
+ decoded_body = response_dict["body"].decode()
840
+ if isinstance(decoded_body, dict):
841
+ error_message = decoded_body.get("message")
842
+ elif isinstance(decoded_body, str):
843
+ error_message = decoded_body
844
+ else:
845
+ error_message = ""
846
+ exception_status = response_dict["headers"].get(":exception-type")
847
+ error_message = exception_status + " " + error_message
848
+ raise BedrockError(
849
+ status_code=response_dict["status_code"],
850
+ message=(
851
+ json.dumps(error_message)
852
+ if isinstance(error_message, dict)
853
+ else error_message
854
+ ),
855
+ )
856
+ if "chunk" in parsed_response:
857
+ chunk = parsed_response.get("chunk")
858
+ if not chunk:
859
+ return None
860
+ return chunk.get("bytes").decode() # type: ignore[no-any-return]
861
+ else:
862
+ chunk = response_dict.get("body")
863
+ if not chunk:
864
+ return None
865
+
866
+ return chunk.decode() # type: ignore[no-any-return]
867
+
868
+
869
+ def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
870
+ """
871
+ Extract anthropic-beta header values and convert them to a list.
872
+ Supports both JSON array format and comma-separated values from user headers.
873
+
874
+ Used by both converse and invoke transformations for consistent handling
875
+ of anthropic-beta headers that should be passed to AWS Bedrock.
876
+
877
+ Args:
878
+ headers (dict): Request headers dictionary
879
+
880
+ Returns:
881
+ List[str]: List of anthropic beta feature strings, empty list if no header
882
+ """
883
+ anthropic_beta_header = headers.get("anthropic-beta")
884
+ if not anthropic_beta_header:
885
+ return []
886
+
887
+ # If it's already a list, return it
888
+ if isinstance(anthropic_beta_header, list):
889
+ return anthropic_beta_header
890
+
891
+ # Try to parse as JSON array first (e.g., '["interleaved-thinking-2025-05-14", "claude-code-20250219"]')
892
+ if isinstance(anthropic_beta_header, str):
893
+ anthropic_beta_header = anthropic_beta_header.strip()
894
+ if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith(
895
+ "]"
896
+ ):
897
+ try:
898
+ parsed = json.loads(anthropic_beta_header)
899
+ if isinstance(parsed, list):
900
+ return [str(beta).strip() for beta in parsed]
901
+ except json.JSONDecodeError:
902
+ pass # Fall through to comma-separated parsing
903
+
904
+ # Fall back to comma-separated values
905
+ return [beta.strip() for beta in anthropic_beta_header.split(",")]
906
+
907
+ return []
908
+
909
+
910
+ class CommonBatchFilesUtils:
911
+ """
912
+ Common utilities for Bedrock batch and file operations.
913
+ Provides shared functionality to reduce code duplication between batches and files.
914
+ """
915
+
916
+ def __init__(self):
917
+ # Import here to avoid circular imports
918
+ from .base_aws_llm import BaseAWSLLM
919
+
920
+ self._base_aws = BaseAWSLLM()
921
+
922
+ def get_bedrock_model_id_from_litellm_model(self, model: str) -> str:
923
+ """
924
+ Extract the actual Bedrock model ID from LiteLLM model name.
925
+
926
+ Args:
927
+ model: LiteLLM model name (e.g., "bedrock/anthropic.claude-3-sonnet-20240229-v1:0")
928
+
929
+ Returns:
930
+ Bedrock model ID (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
931
+ """
932
+ if model.startswith("bedrock/"):
933
+ return model[8:] # Remove "bedrock/" prefix
934
+ return model
935
+
936
+ def parse_s3_uri(self, s3_uri: str) -> tuple:
937
+ """
938
+ Parse S3 URI into bucket and key components.
939
+
940
+ Args:
941
+ s3_uri: S3 URI (e.g., "s3://bucket/key/path")
942
+
943
+ Returns:
944
+ Tuple of (bucket, key)
945
+
946
+ Raises:
947
+ ValueError: If URI format is invalid
948
+ """
949
+ if not s3_uri.startswith("s3://"):
950
+ raise ValueError(f"Invalid S3 URI format: {s3_uri}")
951
+
952
+ s3_parts = s3_uri[5:].split("/", 1) # Remove "s3://" and split on first "/"
953
+ if len(s3_parts) != 2:
954
+ raise ValueError(f"Invalid S3 URI format: {s3_uri}")
955
+
956
+ return s3_parts[0], s3_parts[1] # bucket, key
957
+
958
+ def extract_model_from_s3_file_path(
959
+ self, s3_uri: str, optional_params: dict
960
+ ) -> str:
961
+ """
962
+ Extract model ID from S3 file path.
963
+
964
+ The Bedrock file transformation creates S3 objects with the model name embedded:
965
+ Format: s3://bucket/litellm-bedrock-files-{model}-{uuid}.jsonl
966
+ """
967
+ # Check if model is provided in optional_params first
968
+ if "model" in optional_params and optional_params["model"]:
969
+ return self.get_bedrock_model_id_from_litellm_model(
970
+ optional_params["model"]
971
+ )
972
+
973
+ # Extract model from S3 URI path
974
+ # Expected format: s3://bucket/litellm-bedrock-files-{model}-{uuid}.jsonl
975
+ try:
976
+ bucket, object_key = self.parse_s3_uri(s3_uri)
977
+
978
+ # Extract model from object key if it follows our naming pattern
979
+ if object_key.startswith("litellm-bedrock-files-"):
980
+ # Remove prefix and suffix to get model part
981
+ model_part = object_key[22:] # Remove "litellm-bedrock-files-"
982
+ # Find the last dash before the UUID
983
+ parts = model_part.split("-")
984
+ if len(parts) > 1:
985
+ # Reconstruct model name (everything except the last UUID part and .jsonl)
986
+ model_name = "-".join(parts[:-1])
987
+ if model_name.endswith(".jsonl"):
988
+ model_name = model_name[:-6] # Remove .jsonl
989
+ return model_name
990
+ except Exception:
991
+ pass
992
+
993
+ # Fallback to default model
994
+ return "anthropic.claude-3-5-sonnet-20240620-v1:0"
995
+
996
+ def sign_aws_request(
997
+ self,
998
+ service_name: str,
999
+ data: Union[str, dict, "BedrockCreateBatchRequest"],
1000
+ endpoint_url: str,
1001
+ optional_params: dict,
1002
+ method: str = "POST",
1003
+ ) -> tuple:
1004
+ """
1005
+ Sign AWS request using Signature Version 4.
1006
+
1007
+ Args:
1008
+ service_name: AWS service name ("bedrock" or "s3")
1009
+ data: Request data (string or dict)
1010
+ endpoint_url: Full endpoint URL
1011
+ optional_params: Optional parameters containing AWS credentials
1012
+ method: HTTP method (default: POST)
1013
+
1014
+ Returns:
1015
+ Tuple of (signed_headers, signed_data)
1016
+ """
1017
+ try:
1018
+ from botocore.auth import SigV4Auth
1019
+ from botocore.awsrequest import AWSRequest
1020
+ except ImportError:
1021
+ raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
1022
+
1023
+ # Get AWS credentials using existing methods
1024
+ aws_region_name = self._base_aws._get_aws_region_name(
1025
+ optional_params=optional_params, model=""
1026
+ )
1027
+ credentials = self._base_aws.get_credentials(
1028
+ aws_access_key_id=optional_params.get("aws_access_key_id"),
1029
+ aws_secret_access_key=optional_params.get("aws_secret_access_key"),
1030
+ aws_session_token=optional_params.get("aws_session_token"),
1031
+ aws_region_name=aws_region_name,
1032
+ aws_session_name=optional_params.get("aws_session_name"),
1033
+ aws_profile_name=optional_params.get("aws_profile_name"),
1034
+ aws_role_name=optional_params.get("aws_role_name"),
1035
+ aws_web_identity_token=optional_params.get("aws_web_identity_token"),
1036
+ aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
1037
+ )
1038
+
1039
+ # Prepare the request data
1040
+ method_upper = method.upper()
1041
+ if method_upper == "GET":
1042
+ # GET requests should be signed with an empty payload
1043
+ request_data = ""
1044
+ headers = {}
1045
+ else:
1046
+ if isinstance(data, dict):
1047
+ import json
1048
+
1049
+ request_data = json.dumps(data)
1050
+ else:
1051
+ request_data = data
1052
+ # Prepare headers for non-GET requests
1053
+ headers = {"Content-Type": "application/json"}
1054
+
1055
+ # Create AWS request and sign it
1056
+ sigv4 = SigV4Auth(credentials, service_name, aws_region_name)
1057
+ request = AWSRequest(
1058
+ method=method_upper, url=endpoint_url, data=request_data, headers=headers
1059
+ )
1060
+ sigv4.add_auth(request)
1061
+ prepped = request.prepare()
1062
+
1063
+ return (
1064
+ dict(prepped.headers),
1065
+ request_data.encode("utf-8")
1066
+ if isinstance(request_data, str)
1067
+ else request_data,
1068
+ )
1069
+
1070
+ def generate_unique_job_name(self, model: str, prefix: str = "litellm") -> str:
1071
+ """
1072
+ Generate a unique job name for AWS services.
1073
+ AWS services often have length limits, so this creates a concise name.
1074
+
1075
+ Args:
1076
+ model: Model name to include in the job name
1077
+ prefix: Prefix for the job name
1078
+
1079
+ Returns:
1080
+ Unique job name (≤ 63 characters for Bedrock compatibility)
1081
+ """
1082
+ from litellm._uuid import uuid
1083
+
1084
+ unique_id = str(uuid.uuid4())[:8]
1085
+ # Format: {prefix}-batch-{model}-{uuid}
1086
+ # Example: litellm-batch-claude-266c398e
1087
+ job_name = f"{prefix}-batch-{unique_id}"
1088
+
1089
+ return job_name
1090
+
1091
+ def get_s3_bucket_and_key_from_config(
1092
+ self,
1093
+ litellm_params: dict,
1094
+ optional_params: dict,
1095
+ bucket_env_var: str = "AWS_S3_BUCKET_NAME",
1096
+ key_prefix: str = "litellm",
1097
+ ) -> tuple:
1098
+ """
1099
+ Get S3 bucket and generate a unique key from configuration.
1100
+
1101
+ Args:
1102
+ litellm_params: LiteLLM parameters
1103
+ optional_params: Optional parameters
1104
+ bucket_env_var: Environment variable name for bucket
1105
+ key_prefix: Prefix for the S3 key
1106
+
1107
+ Returns:
1108
+ Tuple of (bucket_name, object_key)
1109
+ """
1110
+ import time
1111
+
1112
+ from litellm._uuid import uuid
1113
+
1114
+ # Get bucket name
1115
+ bucket_name = (
1116
+ litellm_params.get("s3_bucket_name")
1117
+ or optional_params.get("s3_bucket_name")
1118
+ or os.getenv(bucket_env_var)
1119
+ )
1120
+ if not bucket_name:
1121
+ raise ValueError(
1122
+ f"S3 bucket name is required. Set 's3_bucket_name' parameter or {bucket_env_var} env var"
1123
+ )
1124
+
1125
+ # Generate unique object key
1126
+ timestamp = int(time.time())
1127
+ unique_id = str(uuid.uuid4())[:8]
1128
+ object_key = f"{key_prefix}-{timestamp}-{unique_id}"
1129
+
1130
+ return bucket_name, object_key
1131
+
1132
+ def get_error_class(
1133
+ self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers]
1134
+ ) -> BaseLLMException:
1135
+ """
1136
+ Get Bedrock-specific error class.
1137
+ """
1138
+ return BedrockError(
1139
+ status_code=status_code, message=error_message, headers=headers
1140
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/cost_calculation.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Helper util for handling bedrock-specific cost calculation
3
+ - e.g.: prompt caching
4
+ """
5
+
6
+ from typing import TYPE_CHECKING, Optional, Tuple
7
+
8
+ from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
9
+
10
+ if TYPE_CHECKING:
11
+ from litellm.types.utils import Usage
12
+
13
+
14
+ def cost_per_token(
15
+ model: str, usage: "Usage", service_tier: Optional[str] = None
16
+ ) -> Tuple[float, float]:
17
+ """
18
+ Calculates the cost per token for a given model, prompt tokens, and completion tokens.
19
+
20
+ Follows the same logic as Anthropic's cost per token calculation.
21
+ """
22
+ return generic_cost_per_token(
23
+ model=model,
24
+ usage=usage,
25
+ custom_llm_provider="bedrock",
26
+ service_tier=service_tier,
27
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Bedrock Token Counter implementation using the CountTokens API.
3
+ """
4
+
5
+ from typing import Any, Dict, List, Optional
6
+
7
+ from litellm._logging import verbose_logger
8
+ from litellm.llms.base_llm.base_utils import BaseTokenCounter
9
+ from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
10
+ from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
11
+ from litellm.types.utils import LlmProviders, TokenCountResponse
12
+
13
+
14
+ class BedrockTokenCounter(BaseTokenCounter):
15
+ """Token counter implementation for AWS Bedrock provider using the CountTokens API."""
16
+
17
+ def should_use_token_counting_api(
18
+ self,
19
+ custom_llm_provider: Optional[str] = None,
20
+ ) -> bool:
21
+ """
22
+ Returns True if we should use the Bedrock CountTokens API for token counting.
23
+ """
24
+ return custom_llm_provider == LlmProviders.BEDROCK.value
25
+
26
+ async def count_tokens(
27
+ self,
28
+ model_to_use: str,
29
+ messages: Optional[List[Dict[str, Any]]],
30
+ contents: Optional[List[Dict[str, Any]]],
31
+ deployment: Optional[Dict[str, Any]] = None,
32
+ request_model: str = "",
33
+ tools: Optional[List[Dict[str, Any]]] = None,
34
+ system: Optional[Any] = None,
35
+ ) -> Optional[TokenCountResponse]:
36
+ """
37
+ Count tokens using AWS Bedrock's CountTokens API.
38
+
39
+ This method calls the existing BedrockCountTokensHandler to make an API call
40
+ to Bedrock's token counting endpoint, bypassing the local tiktoken-based counting.
41
+
42
+ Args:
43
+ model_to_use: The model identifier
44
+ messages: The messages to count tokens for
45
+ contents: Alternative content format (not used for Bedrock)
46
+ deployment: Deployment configuration containing litellm_params
47
+ request_model: The original request model name
48
+
49
+ Returns:
50
+ TokenCountResponse with token count, or None if counting fails
51
+ """
52
+ if not messages:
53
+ return None
54
+
55
+ deployment = deployment or {}
56
+ litellm_params = deployment.get("litellm_params", {})
57
+
58
+ # Build request data in the format expected by BedrockCountTokensHandler
59
+ request_data: Dict[str, Any] = {
60
+ "model": model_to_use,
61
+ "messages": messages,
62
+ }
63
+
64
+ if tools:
65
+ request_data["tools"] = tools
66
+
67
+ if system:
68
+ request_data["system"] = system
69
+
70
+ # Get the resolved model (strip prefixes like bedrock/, converse/, etc.)
71
+ resolved_model = get_bedrock_base_model(model_to_use)
72
+
73
+ try:
74
+ handler = BedrockCountTokensHandler()
75
+ result = await handler.handle_count_tokens_request(
76
+ request_data=request_data,
77
+ litellm_params=litellm_params,
78
+ resolved_model=resolved_model,
79
+ )
80
+
81
+ # Transform response to TokenCountResponse
82
+ if result is not None:
83
+ return TokenCountResponse(
84
+ total_tokens=result.get("input_tokens", 0),
85
+ request_model=request_model,
86
+ model_used=model_to_use,
87
+ tokenizer_type="bedrock_api",
88
+ original_response=result,
89
+ )
90
+ except BedrockError as e:
91
+ verbose_logger.warning(
92
+ f"Bedrock CountTokens API error: status={e.status_code}, message={e.message}"
93
+ )
94
+ return TokenCountResponse(
95
+ total_tokens=0,
96
+ request_model=request_model,
97
+ model_used=model_to_use,
98
+ tokenizer_type="bedrock_api",
99
+ error=True,
100
+ error_message=e.message,
101
+ status_code=e.status_code,
102
+ )
103
+ except Exception as e:
104
+ verbose_logger.warning(f"Error calling Bedrock CountTokens API: {e}")
105
+ return TokenCountResponse(
106
+ total_tokens=0,
107
+ request_model=request_model,
108
+ model_used=model_to_use,
109
+ tokenizer_type="bedrock_api",
110
+ error=True,
111
+ error_message=str(e),
112
+ status_code=500,
113
+ )
114
+
115
+ return None
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/handler.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ AWS Bedrock CountTokens API handler.
3
+
4
+ Simplified handler leveraging existing LiteLLM Bedrock infrastructure.
5
+ """
6
+
7
+ from typing import Any, Dict
8
+
9
+ import httpx
10
+
11
+ import litellm
12
+ from litellm._logging import verbose_logger
13
+ from litellm.llms.bedrock.common_utils import BedrockError
14
+ from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
15
+ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
16
+
17
+
18
+ class BedrockCountTokensHandler(BedrockCountTokensConfig):
19
+ """
20
+ Simplified handler for AWS Bedrock CountTokens API requests.
21
+
22
+ Uses existing LiteLLM infrastructure for authentication and request handling.
23
+ """
24
+
25
+ async def handle_count_tokens_request(
26
+ self,
27
+ request_data: Dict[str, Any],
28
+ litellm_params: Dict[str, Any],
29
+ resolved_model: str,
30
+ ) -> Dict[str, Any]:
31
+ """
32
+ Handle a CountTokens request using existing LiteLLM patterns.
33
+
34
+ Args:
35
+ request_data: The incoming request payload
36
+ litellm_params: LiteLLM configuration parameters
37
+ resolved_model: The actual model ID resolved from router
38
+
39
+ Returns:
40
+ Dictionary containing token count response
41
+ """
42
+ try:
43
+ # Validate the request
44
+ self.validate_count_tokens_request(request_data)
45
+
46
+ verbose_logger.debug(
47
+ f"Processing CountTokens request for resolved model: {resolved_model}"
48
+ )
49
+
50
+ # Get AWS region using existing LiteLLM function
51
+ aws_region_name = self._get_aws_region_name(
52
+ optional_params=litellm_params,
53
+ model=resolved_model,
54
+ model_id=None,
55
+ )
56
+
57
+ verbose_logger.debug(f"Retrieved AWS region: {aws_region_name}")
58
+
59
+ # Transform request to Bedrock format (supports both Converse and InvokeModel)
60
+ bedrock_request = self.transform_anthropic_to_bedrock_count_tokens(
61
+ request_data=request_data
62
+ )
63
+
64
+ verbose_logger.debug(f"Transformed request: {bedrock_request}")
65
+
66
+ # Get endpoint URL using simplified function
67
+ api_base = litellm_params.get("api_base", None)
68
+ aws_bedrock_runtime_endpoint = litellm_params.get(
69
+ "aws_bedrock_runtime_endpoint", None
70
+ )
71
+ endpoint_url = self.get_bedrock_count_tokens_endpoint(
72
+ model=resolved_model,
73
+ aws_region_name=aws_region_name,
74
+ api_base=api_base,
75
+ aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
76
+ )
77
+
78
+ verbose_logger.debug(f"Making request to: {endpoint_url}")
79
+
80
+ # Use existing _sign_request method from BaseAWSLLM
81
+ # Extract api_key for bearer token auth if provided
82
+ api_key = litellm_params.get("api_key", None)
83
+ headers = {"Content-Type": "application/json"}
84
+ signed_headers, signed_body = self._sign_request(
85
+ service_name="bedrock",
86
+ headers=headers,
87
+ optional_params=litellm_params,
88
+ request_data=bedrock_request,
89
+ api_base=endpoint_url,
90
+ model=resolved_model,
91
+ api_key=api_key,
92
+ )
93
+
94
+ async_client = get_async_httpx_client(
95
+ llm_provider=litellm.LlmProviders.BEDROCK
96
+ )
97
+
98
+ response = await async_client.post(
99
+ endpoint_url,
100
+ headers=signed_headers,
101
+ data=signed_body,
102
+ timeout=30.0,
103
+ )
104
+
105
+ verbose_logger.debug(f"Response status: {response.status_code}")
106
+
107
+ if response.status_code != 200:
108
+ error_text = response.text
109
+ verbose_logger.error(f"AWS Bedrock error: {error_text}")
110
+ raise BedrockError(
111
+ status_code=response.status_code,
112
+ message=error_text,
113
+ )
114
+
115
+ bedrock_response = response.json()
116
+
117
+ verbose_logger.debug(f"Bedrock response: {bedrock_response}")
118
+
119
+ # Transform response back to expected format
120
+ final_response = self.transform_bedrock_response_to_anthropic(
121
+ bedrock_response
122
+ )
123
+
124
+ verbose_logger.debug(f"Final response: {final_response}")
125
+
126
+ return final_response
127
+
128
+ except BedrockError:
129
+ # Re-raise Bedrock exceptions as-is
130
+ raise
131
+ except httpx.HTTPStatusError as e:
132
+ # HTTP errors - preserve the actual status code
133
+ verbose_logger.error(f"HTTP error in CountTokens handler: {str(e)}")
134
+ raise BedrockError(
135
+ status_code=e.response.status_code,
136
+ message=e.response.text,
137
+ )
138
+ except Exception as e:
139
+ verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
140
+ raise BedrockError(
141
+ status_code=500,
142
+ message=f"CountTokens processing error: {str(e)}",
143
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/count_tokens/transformation.py ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ AWS Bedrock CountTokens API transformation logic.
3
+
4
+ This module handles the transformation of requests from Anthropic Messages API format
5
+ to AWS Bedrock's CountTokens API format and vice versa.
6
+ """
7
+
8
+ import re
9
+ from typing import Any, Dict, List, Optional
10
+
11
+ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
12
+ from litellm.llms.bedrock.common_utils import get_bedrock_base_model
13
+
14
+
15
+ class BedrockCountTokensConfig(BaseAWSLLM):
16
+ """
17
+ Configuration and transformation logic for AWS Bedrock CountTokens API.
18
+
19
+ AWS Bedrock CountTokens API Specification:
20
+ - Endpoint: POST /model/{modelId}/count-tokens
21
+ - Input formats: 'invokeModel' or 'converse'
22
+ - Response: {"inputTokens": <number>}
23
+ """
24
+
25
+ def _detect_input_type(self, request_data: Dict[str, Any]) -> str:
26
+ """
27
+ Detect whether to use 'converse' or 'invokeModel' input format.
28
+
29
+ Args:
30
+ request_data: The original request data
31
+
32
+ Returns:
33
+ 'converse' or 'invokeModel'
34
+ """
35
+ # If the request has messages in the expected Anthropic format, use converse
36
+ if "messages" in request_data and isinstance(request_data["messages"], list):
37
+ return "converse"
38
+
39
+ # For raw text or other formats, use invokeModel
40
+ # This handles cases where the input is prompt-based or already in raw Bedrock format
41
+ return "invokeModel"
42
+
43
+ def transform_anthropic_to_bedrock_count_tokens(
44
+ self,
45
+ request_data: Dict[str, Any],
46
+ ) -> Dict[str, Any]:
47
+ """
48
+ Transform request to Bedrock CountTokens format.
49
+ Supports both Converse and InvokeModel input types.
50
+
51
+ Input (Anthropic format):
52
+ {
53
+ "model": "claude-3-5-sonnet",
54
+ "messages": [{"role": "user", "content": "Hello!"}]
55
+ }
56
+
57
+ Output (Bedrock CountTokens format for Converse):
58
+ {
59
+ "input": {
60
+ "converse": {
61
+ "messages": [...],
62
+ "system": [...] (if present)
63
+ }
64
+ }
65
+ }
66
+
67
+ Output (Bedrock CountTokens format for InvokeModel):
68
+ {
69
+ "input": {
70
+ "invokeModel": {
71
+ "body": "{...raw model input...}"
72
+ }
73
+ }
74
+ }
75
+ """
76
+ input_type = self._detect_input_type(request_data)
77
+
78
+ if input_type == "converse":
79
+ return self._transform_to_converse_format(request_data)
80
+ else:
81
+ return self._transform_to_invoke_model_format(request_data)
82
+
83
+ def _transform_to_converse_format(
84
+ self, request_data: Dict[str, Any]
85
+ ) -> Dict[str, Any]:
86
+ """Transform to Converse input format, including system and tools."""
87
+ messages = request_data.get("messages", [])
88
+ system = request_data.get("system")
89
+ tools = request_data.get("tools")
90
+
91
+ # Transform messages
92
+ user_messages = []
93
+ for message in messages:
94
+ transformed_message: Dict[str, Any] = {
95
+ "role": message.get("role"),
96
+ "content": [],
97
+ }
98
+ content = message.get("content", "")
99
+ if isinstance(content, str):
100
+ transformed_message["content"].append({"text": content})
101
+ elif isinstance(content, list):
102
+ transformed_message["content"] = content
103
+ user_messages.append(transformed_message)
104
+
105
+ converse_input: Dict[str, Any] = {"messages": user_messages}
106
+
107
+ # Transform system prompt (string or list of blocks → Bedrock format)
108
+ system_blocks = self._transform_system(system)
109
+ if system_blocks:
110
+ converse_input["system"] = system_blocks
111
+
112
+ # Transform tools (Anthropic format → Bedrock toolConfig)
113
+ tool_config = self._transform_tools(tools)
114
+ if tool_config:
115
+ converse_input["toolConfig"] = tool_config
116
+
117
+ return {"input": {"converse": converse_input}}
118
+
119
+ def _transform_system(self, system: Optional[Any]) -> List[Dict[str, Any]]:
120
+ """Transform Anthropic system prompt to Bedrock system blocks."""
121
+ if system is None:
122
+ return []
123
+ if isinstance(system, str):
124
+ return [{"text": system}]
125
+ if isinstance(system, list):
126
+ # Already in blocks format (e.g. [{"type": "text", "text": "..."}])
127
+ return [
128
+ {"text": block.get("text", "")}
129
+ for block in system
130
+ if isinstance(block, dict)
131
+ ]
132
+ return []
133
+
134
+ def _transform_tools(
135
+ self, tools: Optional[List[Dict[str, Any]]]
136
+ ) -> Optional[Dict[str, Any]]:
137
+ """Transform Anthropic tools to Bedrock toolConfig format."""
138
+ if not tools:
139
+ return None
140
+
141
+ bedrock_tools = []
142
+ for tool in tools:
143
+ name = tool.get("name", "")
144
+ # Bedrock tool names must match [a-zA-Z][a-zA-Z0-9_]* and max 64 chars
145
+ name = re.sub(r"[^a-zA-Z0-9_]", "_", name)
146
+ if name and not name[0].isalpha():
147
+ name = "t_" + name
148
+ name = name[:64]
149
+
150
+ description = tool.get("description") or name
151
+ input_schema = tool.get(
152
+ "input_schema", {"type": "object", "properties": {}}
153
+ )
154
+
155
+ bedrock_tools.append(
156
+ {
157
+ "toolSpec": {
158
+ "name": name,
159
+ "description": description,
160
+ "inputSchema": {"json": input_schema},
161
+ }
162
+ }
163
+ )
164
+
165
+ return {"tools": bedrock_tools}
166
+
167
+ def _transform_to_invoke_model_format(
168
+ self, request_data: Dict[str, Any]
169
+ ) -> Dict[str, Any]:
170
+ """Transform to InvokeModel input format."""
171
+ import json
172
+
173
+ # For InvokeModel, we need to provide the raw body that would be sent to the model
174
+ # Remove the 'model' field from the body as it's not part of the model input
175
+ body_data = {k: v for k, v in request_data.items() if k != "model"}
176
+
177
+ return {"input": {"invokeModel": {"body": json.dumps(body_data)}}}
178
+
179
+ def get_bedrock_count_tokens_endpoint(
180
+ self,
181
+ model: str,
182
+ aws_region_name: str,
183
+ api_base: Optional[str] = None,
184
+ aws_bedrock_runtime_endpoint: Optional[str] = None,
185
+ ) -> str:
186
+ """
187
+ Construct the AWS Bedrock CountTokens API endpoint using existing LiteLLM functions.
188
+
189
+ Args:
190
+ model: The resolved model ID from router lookup
191
+ aws_region_name: AWS region (e.g., "eu-west-1")
192
+ api_base: Optional custom API base URL (takes highest priority)
193
+ aws_bedrock_runtime_endpoint: Optional custom Bedrock runtime endpoint
194
+
195
+ Returns:
196
+ Complete endpoint URL for CountTokens API
197
+ """
198
+ # Use existing LiteLLM function to get the base model ID (removes region prefix)
199
+ model_id = get_bedrock_base_model(model)
200
+
201
+ # Remove bedrock/ prefix if present
202
+ if model_id.startswith("bedrock/"):
203
+ model_id = model_id[8:] # Remove "bedrock/" prefix
204
+
205
+ base_url, _ = self.get_runtime_endpoint(
206
+ api_base=api_base,
207
+ aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
208
+ aws_region_name=aws_region_name,
209
+ )
210
+ endpoint = f"{base_url}/model/{model_id}/count-tokens"
211
+
212
+ return endpoint
213
+
214
+ def transform_bedrock_response_to_anthropic(
215
+ self, bedrock_response: Dict[str, Any]
216
+ ) -> Dict[str, Any]:
217
+ """
218
+ Transform Bedrock CountTokens response to Anthropic format.
219
+
220
+ Input (Bedrock response):
221
+ {
222
+ "inputTokens": 123
223
+ }
224
+
225
+ Output (Anthropic format):
226
+ {
227
+ "input_tokens": 123
228
+ }
229
+ """
230
+ input_tokens = bedrock_response.get("inputTokens", 0)
231
+
232
+ return {"input_tokens": input_tokens}
233
+
234
+ def validate_count_tokens_request(self, request_data: Dict[str, Any]) -> None:
235
+ """
236
+ Validate the incoming count tokens request.
237
+ Supports both Converse and InvokeModel input formats.
238
+
239
+ Args:
240
+ request_data: The request payload
241
+
242
+ Raises:
243
+ ValueError: If the request is invalid
244
+ """
245
+ if not request_data.get("model"):
246
+ raise ValueError("model parameter is required")
247
+
248
+ input_type = self._detect_input_type(request_data)
249
+
250
+ if input_type == "converse":
251
+ # Validate Converse format (messages-based)
252
+ messages = request_data.get("messages", [])
253
+ if not messages:
254
+ raise ValueError("messages parameter is required for Converse input")
255
+
256
+ if not isinstance(messages, list):
257
+ raise ValueError("messages must be a list")
258
+
259
+ for i, message in enumerate(messages):
260
+ if not isinstance(message, dict):
261
+ raise ValueError(f"Message {i} must be a dictionary")
262
+
263
+ if "role" not in message:
264
+ raise ValueError(f"Message {i} must have a 'role' field")
265
+
266
+ if "content" not in message:
267
+ raise ValueError(f"Message {i} must have a 'content' field")
268
+ else:
269
+ # For InvokeModel format, we need at least some content to count tokens
270
+ # The content structure varies by model, so we do minimal validation
271
+ if len(request_data) <= 1: # Only has 'model' field
272
+ raise ValueError("Request must contain content to count tokens")
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_nova_transformation.py ADDED
@@ -0,0 +1,361 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Nova /invoke and /async-invoke format.
3
+
4
+ Why separate file? Make it easy to see how transformation works
5
+
6
+ Supports:
7
+ - Synchronous embeddings (SINGLE_EMBEDDING)
8
+ - Asynchronous embeddings with segmentation (SEGMENTED_EMBEDDING)
9
+ - Multimodal inputs: text, image, video, audio
10
+ - Multiple embedding purposes and dimensions
11
+
12
+ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html
13
+ """
14
+
15
+ from typing import List, Optional
16
+
17
+ from litellm.types.utils import (
18
+ Embedding,
19
+ EmbeddingResponse,
20
+ PromptTokensDetailsWrapper,
21
+ Usage,
22
+ )
23
+
24
+
25
+ class AmazonNovaEmbeddingConfig:
26
+ """
27
+ Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html
28
+
29
+ Amazon Nova Multimodal Embeddings supports:
30
+ - Text, image, video, and audio inputs
31
+ - Synchronous (InvokeModel) and asynchronous (StartAsyncInvoke) APIs
32
+ - Multiple embedding purposes and dimensions
33
+ """
34
+
35
+ def __init__(self) -> None:
36
+ pass
37
+
38
+ def get_supported_openai_params(self) -> List[str]:
39
+ return [
40
+ "dimensions",
41
+ ]
42
+
43
+ def map_openai_params(
44
+ self, non_default_params: dict, optional_params: dict
45
+ ) -> dict:
46
+ """Map OpenAI-style parameters to Nova parameters."""
47
+ for k, v in non_default_params.items():
48
+ if k == "dimensions":
49
+ # Map OpenAI dimensions to Nova embedding_dimension
50
+ optional_params["embedding_dimension"] = v
51
+ elif k in self.get_supported_openai_params():
52
+ optional_params[k] = v
53
+ return optional_params
54
+
55
+ def _parse_data_url(self, data_url: str) -> tuple:
56
+ """
57
+ Parse a data URL to extract the media type and base64 data.
58
+
59
+ Args:
60
+ data_url: Data URL in format: data:image/jpeg;base64,/9j/4AAQ...
61
+
62
+ Returns:
63
+ tuple: (media_type, base64_data)
64
+ media_type: e.g., "image/jpeg", "video/mp4", "audio/mpeg"
65
+ base64_data: The base64-encoded data without the prefix
66
+ """
67
+ if not data_url.startswith("data:"):
68
+ raise ValueError(f"Invalid data URL format: {data_url[:50]}...")
69
+
70
+ # Split by comma to separate metadata from data
71
+ # Format: data:image/jpeg;base64,<base64_data>
72
+ if "," not in data_url:
73
+ raise ValueError(
74
+ f"Invalid data URL format (missing comma): {data_url[:50]}..."
75
+ )
76
+
77
+ metadata, base64_data = data_url.split(",", 1)
78
+
79
+ # Extract media type from metadata
80
+ # Remove 'data:' prefix and ';base64' suffix
81
+ metadata = metadata[5:] # Remove 'data:'
82
+
83
+ if ";" in metadata:
84
+ media_type = metadata.split(";")[0]
85
+ else:
86
+ media_type = metadata
87
+
88
+ return media_type, base64_data
89
+
90
+ def _transform_request(
91
+ self,
92
+ input: str,
93
+ inference_params: dict,
94
+ async_invoke_route: bool = False,
95
+ model_id: Optional[str] = None,
96
+ output_s3_uri: Optional[str] = None,
97
+ ) -> dict:
98
+ """
99
+ Transform OpenAI-style input to Nova format.
100
+
101
+ Only handles OpenAI params (dimensions). All other Nova-specific params
102
+ should be passed via inference_params and will be passed through as-is.
103
+
104
+ Args:
105
+ input: The input text or media reference
106
+ inference_params: Additional parameters (will be passed through)
107
+ async_invoke_route: Whether this is for async invoke
108
+ model_id: Model ID (for async invoke)
109
+ output_s3_uri: S3 URI for output (for async invoke)
110
+
111
+ Returns:
112
+ dict: Nova embedding request
113
+ """
114
+ # Determine task type
115
+ task_type = "SEGMENTED_EMBEDDING" if async_invoke_route else "SINGLE_EMBEDDING"
116
+
117
+ # Build the base request structure
118
+ request: dict = {
119
+ "schemaVersion": "nova-multimodal-embed-v1",
120
+ "taskType": task_type,
121
+ }
122
+
123
+ # Start with inference_params (user-provided params)
124
+ embedding_params = inference_params.copy()
125
+
126
+ embedding_params.pop("output_s3_uri", None)
127
+
128
+ # Map OpenAI dimensions to embeddingDimension if provided
129
+ if "dimensions" in embedding_params:
130
+ embedding_params["embeddingDimension"] = embedding_params.pop("dimensions")
131
+ elif "embedding_dimension" in embedding_params:
132
+ embedding_params["embeddingDimension"] = embedding_params.pop(
133
+ "embedding_dimension"
134
+ )
135
+
136
+ # Add required embeddingPurpose if not provided (required by Nova API)
137
+ if "embeddingPurpose" not in embedding_params:
138
+ embedding_params["embeddingPurpose"] = "GENERIC_INDEX"
139
+
140
+ # Add required embeddingDimension if not provided (required by Nova API)
141
+ if "embeddingDimension" not in embedding_params:
142
+ embedding_params["embeddingDimension"] = 3072
143
+
144
+ # For text/media input, add basic structure if user hasn't provided text/image/video/audio
145
+ if (
146
+ "text" not in embedding_params
147
+ and "image" not in embedding_params
148
+ and "video" not in embedding_params
149
+ and "audio" not in embedding_params
150
+ ):
151
+ # Check if input is a data URL (e.g., data:image/jpeg;base64,...)
152
+ if input.startswith("data:"):
153
+ # Parse the data URL to extract media type and base64 data
154
+ media_type, base64_data = self._parse_data_url(input)
155
+
156
+ if media_type.startswith("image/"):
157
+ # Extract image format from MIME type (e.g., image/jpeg -> jpeg)
158
+ image_format = media_type.split("/")[1].lower()
159
+ # Nova API expects specific formats
160
+ if image_format == "jpg":
161
+ image_format = "jpeg"
162
+
163
+ embedding_params["image"] = {
164
+ "format": image_format,
165
+ "source": {"bytes": base64_data},
166
+ }
167
+ elif media_type.startswith("video/"):
168
+ # Handle video data URLs
169
+ video_format = media_type.split("/")[1].lower()
170
+ embedding_params["video"] = {
171
+ "format": video_format,
172
+ "source": {"bytes": base64_data},
173
+ }
174
+ elif media_type.startswith("audio/"):
175
+ # Handle audio data URLs
176
+ audio_format = media_type.split("/")[1].lower()
177
+ embedding_params["audio"] = {
178
+ "format": audio_format,
179
+ "source": {"bytes": base64_data},
180
+ }
181
+ else:
182
+ # Fallback to text for unknown types
183
+ embedding_params["text"] = {"value": input, "truncationMode": "END"}
184
+ elif input.startswith("s3://"):
185
+ # S3 URL - default to text for now, user should specify modality
186
+ embedding_params["text"] = {
187
+ "source": {"s3Location": {"uri": input}},
188
+ "truncationMode": "END", # Required by Nova API
189
+ }
190
+ else:
191
+ # Plain text input
192
+ embedding_params["text"] = {
193
+ "value": input,
194
+ "truncationMode": "END", # Required by Nova API
195
+ }
196
+
197
+ # Set the embedding params in the request
198
+ if task_type == "SINGLE_EMBEDDING":
199
+ request["singleEmbeddingParams"] = embedding_params
200
+ else:
201
+ request["segmentedEmbeddingParams"] = embedding_params
202
+
203
+ # For async invoke, wrap in the async invoke format
204
+ if async_invoke_route and model_id:
205
+ return self._wrap_async_invoke_request(
206
+ model_input=request,
207
+ model_id=model_id,
208
+ output_s3_uri=output_s3_uri,
209
+ )
210
+
211
+ return request
212
+
213
+ def _wrap_async_invoke_request(
214
+ self,
215
+ model_input: dict,
216
+ model_id: str,
217
+ output_s3_uri: Optional[str] = None,
218
+ ) -> dict:
219
+ """
220
+ Wrap the transformed request in the AWS Bedrock async invoke format.
221
+
222
+ Args:
223
+ model_input: The transformed Nova embedding request
224
+ model_id: The model identifier (without async_invoke prefix)
225
+ output_s3_uri: S3 URI for output data config
226
+
227
+ Returns:
228
+ dict: The wrapped async invoke request
229
+ """
230
+ import urllib.parse
231
+
232
+ # Clean the model ID
233
+ unquoted_model_id = urllib.parse.unquote(model_id)
234
+ if unquoted_model_id.startswith("async_invoke/"):
235
+ unquoted_model_id = unquoted_model_id.replace("async_invoke/", "")
236
+
237
+ # Validate that the S3 URI is not empty
238
+ if not output_s3_uri or output_s3_uri.strip() == "":
239
+ raise ValueError("output_s3_uri is required for async invoke requests")
240
+
241
+ return {
242
+ "modelId": unquoted_model_id,
243
+ "modelInput": model_input,
244
+ "outputDataConfig": {"s3OutputDataConfig": {"s3Uri": output_s3_uri}},
245
+ }
246
+
247
+ def _transform_response(
248
+ self,
249
+ response_list: List[dict],
250
+ model: str,
251
+ batch_data: Optional[List[dict]] = None,
252
+ ) -> EmbeddingResponse:
253
+ """
254
+ Transform Nova response to OpenAI format.
255
+
256
+ Nova response format:
257
+ {
258
+ "embeddings": [
259
+ {
260
+ "embeddingType": "TEXT" | "IMAGE" | "VIDEO" | "AUDIO" | "AUDIO_VIDEO_COMBINED",
261
+ "embedding": [0.1, 0.2, ...],
262
+ "truncatedCharLength": 100 # Optional, only for text
263
+ }
264
+ ]
265
+ }
266
+ """
267
+ embeddings: List[Embedding] = []
268
+ total_tokens = 0
269
+
270
+ for response in response_list:
271
+ # Nova response has an "embeddings" array
272
+ if "embeddings" in response and isinstance(response["embeddings"], list):
273
+ for item in response["embeddings"]:
274
+ if "embedding" in item:
275
+ embedding = Embedding(
276
+ embedding=item["embedding"],
277
+ index=len(embeddings),
278
+ object="embedding",
279
+ )
280
+ embeddings.append(embedding)
281
+
282
+ # Estimate token count
283
+ # For text, use truncatedCharLength if available
284
+ if "truncatedCharLength" in item:
285
+ total_tokens += item["truncatedCharLength"] // 4
286
+ else:
287
+ # Rough estimate based on embedding dimension
288
+ total_tokens += len(item["embedding"]) // 4
289
+ elif "embedding" in response:
290
+ # Direct embedding response (fallback)
291
+ embedding = Embedding(
292
+ embedding=response["embedding"],
293
+ index=len(embeddings),
294
+ object="embedding",
295
+ )
296
+ embeddings.append(embedding)
297
+ total_tokens += len(response["embedding"]) // 4
298
+
299
+ # Count images from original requests for cost calculation
300
+ image_count = 0
301
+ if batch_data:
302
+ for request_data in batch_data:
303
+ # Nova wraps params in singleEmbeddingParams or segmentedEmbeddingParams
304
+ params = request_data.get(
305
+ "singleEmbeddingParams",
306
+ request_data.get("segmentedEmbeddingParams", {}),
307
+ )
308
+ if "image" in params:
309
+ image_count += 1
310
+
311
+ prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
312
+ if image_count > 0:
313
+ prompt_tokens_details = PromptTokensDetailsWrapper(
314
+ image_count=image_count,
315
+ )
316
+
317
+ usage = Usage(
318
+ prompt_tokens=total_tokens,
319
+ total_tokens=total_tokens,
320
+ prompt_tokens_details=prompt_tokens_details,
321
+ )
322
+
323
+ return EmbeddingResponse(data=embeddings, model=model, usage=usage)
324
+
325
+ def _transform_async_invoke_response(
326
+ self, response: dict, model: str
327
+ ) -> EmbeddingResponse:
328
+ """
329
+ Transform async invoke response (invocation ARN) to OpenAI format.
330
+
331
+ AWS async invoke returns:
332
+ {
333
+ "invocationArn": "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"
334
+ }
335
+
336
+ We transform this to a job-like embedding response with the ARN in hidden params.
337
+ """
338
+ invocation_arn = response.get("invocationArn", "")
339
+
340
+ # Create a placeholder embedding object for the job
341
+ embedding = Embedding(
342
+ embedding=[], # Empty embedding for async jobs
343
+ index=0,
344
+ object="embedding",
345
+ )
346
+
347
+ # Create usage object (empty for async jobs)
348
+ usage = Usage(prompt_tokens=0, total_tokens=0)
349
+
350
+ # Create hidden params with job ID
351
+ from litellm.types.llms.base import HiddenParams
352
+
353
+ hidden_params = HiddenParams()
354
+ setattr(hidden_params, "_invocation_arn", invocation_arn)
355
+
356
+ return EmbeddingResponse(
357
+ data=[embedding],
358
+ model=model,
359
+ usage=usage,
360
+ hidden_params=hidden_params,
361
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Titan G1 /invoke format.
3
+
4
+ Why separate file? Make it easy to see how transformation works
5
+
6
+ Convers
7
+ - G1 request format
8
+
9
+ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html
10
+ """
11
+
12
+ import types
13
+ from typing import List
14
+
15
+ from litellm.types.llms.bedrock import (
16
+ AmazonTitanG1EmbeddingRequest,
17
+ AmazonTitanG1EmbeddingResponse,
18
+ )
19
+ from litellm.types.utils import Embedding, EmbeddingResponse, Usage
20
+
21
+
22
+ class AmazonTitanG1Config:
23
+ """
24
+ Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html
25
+ """
26
+
27
+ def __init__(
28
+ self,
29
+ ) -> None:
30
+ locals_ = locals().copy()
31
+ for key, value in locals_.items():
32
+ if key != "self" and value is not None:
33
+ setattr(self.__class__, key, value)
34
+
35
+ @classmethod
36
+ def get_config(cls):
37
+ return {
38
+ k: v
39
+ for k, v in cls.__dict__.items()
40
+ if not k.startswith("__")
41
+ and not isinstance(
42
+ v,
43
+ (
44
+ types.FunctionType,
45
+ types.BuiltinFunctionType,
46
+ classmethod,
47
+ staticmethod,
48
+ ),
49
+ )
50
+ and v is not None
51
+ }
52
+
53
+ def get_supported_openai_params(self) -> List[str]:
54
+ return []
55
+
56
+ def map_openai_params(
57
+ self, non_default_params: dict, optional_params: dict
58
+ ) -> dict:
59
+ return optional_params
60
+
61
+ def _transform_request(
62
+ self, input: str, inference_params: dict
63
+ ) -> AmazonTitanG1EmbeddingRequest:
64
+ return AmazonTitanG1EmbeddingRequest(inputText=input)
65
+
66
+ def _transform_response(
67
+ self, response_list: List[dict], model: str
68
+ ) -> EmbeddingResponse:
69
+ total_prompt_tokens = 0
70
+
71
+ transformed_responses: List[Embedding] = []
72
+ for index, response in enumerate(response_list):
73
+ _parsed_response = AmazonTitanG1EmbeddingResponse(**response) # type: ignore
74
+ transformed_responses.append(
75
+ Embedding(
76
+ embedding=_parsed_response["embedding"],
77
+ index=index,
78
+ object="embedding",
79
+ )
80
+ )
81
+ total_prompt_tokens += _parsed_response["inputTextTokenCount"]
82
+
83
+ usage = Usage(
84
+ prompt_tokens=total_prompt_tokens,
85
+ completion_tokens=0,
86
+ total_tokens=total_prompt_tokens,
87
+ )
88
+ return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Titan multimodal /invoke format.
3
+
4
+ Why separate file? Make it easy to see how transformation works
5
+
6
+ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-mm.html
7
+ """
8
+
9
+ from typing import List, Optional
10
+
11
+ from litellm.types.llms.bedrock import (
12
+ AmazonTitanMultimodalEmbeddingConfig,
13
+ AmazonTitanMultimodalEmbeddingRequest,
14
+ AmazonTitanMultimodalEmbeddingResponse,
15
+ )
16
+ from litellm.types.utils import (
17
+ Embedding,
18
+ EmbeddingResponse,
19
+ PromptTokensDetailsWrapper,
20
+ Usage,
21
+ )
22
+ from litellm.utils import get_base64_str, is_base64_encoded
23
+
24
+
25
+ class AmazonTitanMultimodalEmbeddingG1Config:
26
+ """
27
+ Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-mm.html
28
+ """
29
+
30
+ def __init__(self) -> None:
31
+ pass
32
+
33
+ def get_supported_openai_params(self) -> List[str]:
34
+ return ["dimensions"]
35
+
36
+ def map_openai_params(
37
+ self, non_default_params: dict, optional_params: dict
38
+ ) -> dict:
39
+ for k, v in non_default_params.items():
40
+ if k == "dimensions":
41
+ optional_params[
42
+ "embeddingConfig"
43
+ ] = AmazonTitanMultimodalEmbeddingConfig(outputEmbeddingLength=v)
44
+ return optional_params
45
+
46
+ def _transform_request(
47
+ self, input: str, inference_params: dict
48
+ ) -> AmazonTitanMultimodalEmbeddingRequest:
49
+ ## check if b64 encoded str or not ##
50
+ is_encoded = is_base64_encoded(input)
51
+ if is_encoded: # check if string is b64 encoded image or not
52
+ b64_str = get_base64_str(input)
53
+ transformed_request = AmazonTitanMultimodalEmbeddingRequest(
54
+ inputImage=b64_str
55
+ )
56
+ else:
57
+ transformed_request = AmazonTitanMultimodalEmbeddingRequest(inputText=input)
58
+
59
+ for k, v in inference_params.items():
60
+ transformed_request[k] = v # type: ignore
61
+ return transformed_request
62
+
63
+ def _transform_response(
64
+ self,
65
+ response_list: List[dict],
66
+ model: str,
67
+ batch_data: Optional[List[dict]] = None,
68
+ ) -> EmbeddingResponse:
69
+ total_prompt_tokens = 0
70
+ transformed_responses: List[Embedding] = []
71
+ for index, response in enumerate(response_list):
72
+ _parsed_response = AmazonTitanMultimodalEmbeddingResponse(**response) # type: ignore
73
+ transformed_responses.append(
74
+ Embedding(
75
+ embedding=_parsed_response["embedding"],
76
+ index=index,
77
+ object="embedding",
78
+ )
79
+ )
80
+ total_prompt_tokens += _parsed_response["inputTextTokenCount"]
81
+
82
+ # Count images from original requests for cost calculation
83
+ image_count = 0
84
+ if batch_data:
85
+ for request_data in batch_data:
86
+ if "inputImage" in request_data:
87
+ image_count += 1
88
+
89
+ prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
90
+ if image_count > 0:
91
+ prompt_tokens_details = PromptTokensDetailsWrapper(
92
+ image_count=image_count,
93
+ )
94
+
95
+ usage = Usage(
96
+ prompt_tokens=total_prompt_tokens,
97
+ completion_tokens=0,
98
+ total_tokens=total_prompt_tokens,
99
+ prompt_tokens_details=prompt_tokens_details,
100
+ )
101
+ return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Titan V2 /invoke format.
3
+
4
+ Why separate file? Make it easy to see how transformation works
5
+
6
+ Convers
7
+ - v2 request format
8
+
9
+ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html
10
+ """
11
+
12
+ import types
13
+ from typing import List, Optional, Union
14
+
15
+ from litellm.types.llms.bedrock import (
16
+ AmazonTitanV2EmbeddingRequest,
17
+ AmazonTitanV2EmbeddingResponse,
18
+ )
19
+ from litellm.types.utils import Embedding, EmbeddingResponse, Usage
20
+
21
+
22
+ class AmazonTitanV2Config:
23
+ """
24
+ Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html
25
+
26
+ normalize: boolean - flag indicating whether or not to normalize the output embeddings. Defaults to true
27
+ dimensions: int - The number of dimensions the output embeddings should have. The following values are accepted: 1024 (default), 512, 256.
28
+ """
29
+
30
+ normalize: Optional[bool] = None
31
+ dimensions: Optional[int] = None
32
+
33
+ def __init__(
34
+ self, normalize: Optional[bool] = None, dimensions: Optional[int] = None
35
+ ) -> None:
36
+ locals_ = locals().copy()
37
+ for key, value in locals_.items():
38
+ if key != "self" and value is not None:
39
+ setattr(self.__class__, key, value)
40
+
41
+ @classmethod
42
+ def get_config(cls):
43
+ return {
44
+ k: v
45
+ for k, v in cls.__dict__.items()
46
+ if not k.startswith("__")
47
+ and not isinstance(
48
+ v,
49
+ (
50
+ types.FunctionType,
51
+ types.BuiltinFunctionType,
52
+ classmethod,
53
+ staticmethod,
54
+ ),
55
+ )
56
+ and v is not None
57
+ }
58
+
59
+ def get_supported_openai_params(self) -> List[str]:
60
+ return ["dimensions", "encoding_format"]
61
+
62
+ def map_openai_params(
63
+ self, non_default_params: dict, optional_params: dict
64
+ ) -> dict:
65
+ for k, v in non_default_params.items():
66
+ if k == "dimensions":
67
+ optional_params["dimensions"] = v
68
+ elif k == "encoding_format":
69
+ # Map OpenAI encoding_format to AWS embeddingTypes
70
+ if v == "float":
71
+ optional_params["embeddingTypes"] = ["float"]
72
+ elif v == "base64":
73
+ # base64 maps to binary format in AWS
74
+ optional_params["embeddingTypes"] = ["binary"]
75
+ else:
76
+ # For any other encoding format, default to float
77
+ optional_params["embeddingTypes"] = ["float"]
78
+ return optional_params
79
+
80
+ def _transform_request(
81
+ self, input: str, inference_params: dict
82
+ ) -> AmazonTitanV2EmbeddingRequest:
83
+ return AmazonTitanV2EmbeddingRequest(inputText=input, **inference_params) # type: ignore
84
+
85
+ def _transform_response(
86
+ self, response_list: List[dict], model: str
87
+ ) -> EmbeddingResponse:
88
+ total_prompt_tokens = 0
89
+
90
+ transformed_responses: List[Embedding] = []
91
+ for index, response in enumerate(response_list):
92
+ _parsed_response = AmazonTitanV2EmbeddingResponse(**response) # type: ignore
93
+
94
+ # According to AWS docs, embeddingsByType is always present
95
+ # If binary was requested (encoding_format="base64"), use binary data
96
+ # Otherwise, use float data from embeddingsByType or fallback to embedding field
97
+ embedding_data: Union[List[float], List[int]]
98
+
99
+ if (
100
+ "embeddingsByType" in _parsed_response
101
+ and "binary" in _parsed_response["embeddingsByType"]
102
+ ):
103
+ # Use binary data if available (for encoding_format="base64")
104
+ embedding_data = _parsed_response["embeddingsByType"]["binary"]
105
+ elif (
106
+ "embeddingsByType" in _parsed_response
107
+ and "float" in _parsed_response["embeddingsByType"]
108
+ ):
109
+ # Use float data from embeddingsByType
110
+ embedding_data = _parsed_response["embeddingsByType"]["float"]
111
+ elif "embedding" in _parsed_response:
112
+ # Fallback to legacy embedding field
113
+ embedding_data = _parsed_response["embedding"]
114
+ else:
115
+ raise ValueError(f"No embedding data found in response: {response}")
116
+
117
+ transformed_responses.append(
118
+ Embedding(
119
+ embedding=embedding_data,
120
+ index=index,
121
+ object="embedding",
122
+ )
123
+ )
124
+ total_prompt_tokens += _parsed_response["inputTextTokenCount"]
125
+
126
+ usage = Usage(
127
+ prompt_tokens=total_prompt_tokens,
128
+ completion_tokens=0,
129
+ total_tokens=total_prompt_tokens,
130
+ )
131
+ return EmbeddingResponse(model=model, usage=usage, data=transformed_responses)
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/cohere_transformation.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation logic from OpenAI /v1/embeddings format to Bedrock Cohere /invoke format.
3
+
4
+ Why separate file? Make it easy to see how transformation works
5
+ """
6
+
7
+ from typing import List
8
+
9
+ from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig
10
+ from litellm.types.llms.bedrock import CohereEmbeddingRequest
11
+
12
+
13
+ class BedrockCohereEmbeddingConfig:
14
+ def __init__(self) -> None:
15
+ pass
16
+
17
+ def get_supported_openai_params(self) -> List[str]:
18
+ return ["encoding_format", "dimensions"]
19
+
20
+ def map_openai_params(
21
+ self, non_default_params: dict, optional_params: dict
22
+ ) -> dict:
23
+ for k, v in non_default_params.items():
24
+ if k == "encoding_format":
25
+ optional_params["embedding_types"] = v
26
+ elif k == "dimensions":
27
+ optional_params["output_dimension"] = v
28
+ return optional_params
29
+
30
+ def _is_v3_model(self, model: str) -> bool:
31
+ return "3" in model
32
+
33
+ def _transform_request(
34
+ self, model: str, input: List[str], inference_params: dict
35
+ ) -> CohereEmbeddingRequest:
36
+ transformed_request = CohereEmbeddingConfig()._transform_request(
37
+ model, input, inference_params
38
+ )
39
+
40
+ new_transformed_request = CohereEmbeddingRequest(
41
+ input_type=transformed_request["input_type"],
42
+ )
43
+ for k in CohereEmbeddingRequest.__annotations__.keys():
44
+ if k in transformed_request:
45
+ new_transformed_request[k] = transformed_request[k] # type: ignore
46
+
47
+ return new_transformed_request
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/embedding.py ADDED
@@ -0,0 +1,699 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Handles embedding calls to Bedrock's `/invoke` endpoint
3
+ """
4
+
5
+ import copy
6
+ import json
7
+ import urllib.parse
8
+ from typing import Any, Callable, List, Optional, Tuple, Union, get_args
9
+
10
+ import httpx
11
+
12
+ import litellm
13
+ from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
14
+ from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
15
+ from litellm.llms.custom_httpx.http_handler import (
16
+ AsyncHTTPHandler,
17
+ HTTPHandler,
18
+ _get_httpx_client,
19
+ get_async_httpx_client,
20
+ )
21
+ from litellm.secret_managers.main import get_secret
22
+ from litellm.types.llms.bedrock import (
23
+ AmazonEmbeddingRequest,
24
+ CohereEmbeddingRequest,
25
+ )
26
+ from litellm.types.utils import EmbeddingResponse, LlmProviders
27
+
28
+ from ..base_aws_llm import BaseAWSLLM
29
+ from ..common_utils import BedrockError
30
+ from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
31
+ from .amazon_titan_g1_transformation import AmazonTitanG1Config
32
+ from .amazon_titan_multimodal_transformation import (
33
+ AmazonTitanMultimodalEmbeddingG1Config,
34
+ )
35
+ from .amazon_titan_v2_transformation import AmazonTitanV2Config
36
+ from .cohere_transformation import BedrockCohereEmbeddingConfig
37
+ from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig
38
+
39
+
40
+ class BedrockEmbedding(BaseAWSLLM):
41
+ def _load_credentials(
42
+ self,
43
+ optional_params: dict,
44
+ ) -> Tuple[Any, str]:
45
+ try:
46
+ from botocore.credentials import Credentials
47
+ except ImportError:
48
+ raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
49
+ ## CREDENTIALS ##
50
+ # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them
51
+ aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
52
+ aws_access_key_id = optional_params.pop("aws_access_key_id", None)
53
+ aws_session_token = optional_params.pop("aws_session_token", None)
54
+ aws_region_name = optional_params.pop("aws_region_name", None)
55
+ aws_role_name = optional_params.pop("aws_role_name", None)
56
+ aws_session_name = optional_params.pop("aws_session_name", None)
57
+ aws_profile_name = optional_params.pop("aws_profile_name", None)
58
+ aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
59
+ aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
60
+
61
+ ### SET REGION NAME ###
62
+ if aws_region_name is None:
63
+ # check env #
64
+ litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
65
+
66
+ if litellm_aws_region_name is not None and isinstance(
67
+ litellm_aws_region_name, str
68
+ ):
69
+ aws_region_name = litellm_aws_region_name
70
+
71
+ standard_aws_region_name = get_secret("AWS_REGION", None)
72
+ if standard_aws_region_name is not None and isinstance(
73
+ standard_aws_region_name, str
74
+ ):
75
+ aws_region_name = standard_aws_region_name
76
+
77
+ if aws_region_name is None:
78
+ aws_region_name = "us-west-2"
79
+
80
+ credentials: Credentials = self.get_credentials( # type: ignore
81
+ aws_access_key_id=aws_access_key_id,
82
+ aws_secret_access_key=aws_secret_access_key,
83
+ aws_session_token=aws_session_token,
84
+ aws_region_name=aws_region_name,
85
+ aws_session_name=aws_session_name,
86
+ aws_profile_name=aws_profile_name,
87
+ aws_role_name=aws_role_name,
88
+ aws_web_identity_token=aws_web_identity_token,
89
+ aws_sts_endpoint=aws_sts_endpoint,
90
+ )
91
+ return credentials, aws_region_name
92
+
93
+ async def async_embeddings(self):
94
+ pass
95
+
96
+ def _make_sync_call(
97
+ self,
98
+ client: Optional[HTTPHandler],
99
+ timeout: Optional[Union[float, httpx.Timeout]],
100
+ api_base: str,
101
+ headers: dict,
102
+ data: dict,
103
+ ) -> dict:
104
+ if client is None or not isinstance(client, HTTPHandler):
105
+ _params = {}
106
+ if timeout is not None:
107
+ if isinstance(timeout, float) or isinstance(timeout, int):
108
+ timeout = httpx.Timeout(timeout)
109
+ _params["timeout"] = timeout
110
+ client = _get_httpx_client(_params) # type: ignore
111
+ else:
112
+ client = client
113
+ try:
114
+ response = client.post(url=api_base, headers=headers, data=json.dumps(data)) # type: ignore
115
+ response.raise_for_status()
116
+ except httpx.HTTPStatusError as err:
117
+ error_code = err.response.status_code
118
+ raise BedrockError(status_code=error_code, message=err.response.text)
119
+ except httpx.TimeoutException:
120
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
121
+
122
+ return response.json()
123
+
124
+ async def _make_async_call(
125
+ self,
126
+ client: Optional[AsyncHTTPHandler],
127
+ timeout: Optional[Union[float, httpx.Timeout]],
128
+ api_base: str,
129
+ headers: dict,
130
+ data: dict,
131
+ ) -> dict:
132
+ if client is None or not isinstance(client, AsyncHTTPHandler):
133
+ _params = {}
134
+ if timeout is not None:
135
+ if isinstance(timeout, float) or isinstance(timeout, int):
136
+ timeout = httpx.Timeout(timeout)
137
+ _params["timeout"] = timeout
138
+ client = get_async_httpx_client(
139
+ params=_params, llm_provider=litellm.LlmProviders.BEDROCK
140
+ )
141
+ else:
142
+ client = client
143
+
144
+ try:
145
+ response = await client.post(url=api_base, headers=headers, data=json.dumps(data)) # type: ignore
146
+ response.raise_for_status()
147
+ except httpx.HTTPStatusError as err:
148
+ error_code = err.response.status_code
149
+ raise BedrockError(status_code=error_code, message=err.response.text)
150
+ except httpx.TimeoutException:
151
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
152
+
153
+ return response.json()
154
+
155
+ def _transform_response(
156
+ self,
157
+ response_list: List[dict],
158
+ model: str,
159
+ provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
160
+ is_async_invoke: Optional[bool] = False,
161
+ batch_data: Optional[List[dict]] = None,
162
+ ) -> Optional[EmbeddingResponse]:
163
+ """
164
+ Transforms the response from the Bedrock embedding provider to the OpenAI format.
165
+ """
166
+ returned_response: Optional[EmbeddingResponse] = None
167
+
168
+ # Handle async invoke responses (single response with invocationArn)
169
+ if (
170
+ is_async_invoke
171
+ and len(response_list) == 1
172
+ and "invocationArn" in response_list[0]
173
+ ):
174
+ if provider == "twelvelabs":
175
+ returned_response = (
176
+ TwelveLabsMarengoEmbeddingConfig()._transform_async_invoke_response(
177
+ response=response_list[0], model=model
178
+ )
179
+ )
180
+ elif provider == "nova":
181
+ returned_response = (
182
+ AmazonNovaEmbeddingConfig()._transform_async_invoke_response(
183
+ response=response_list[0], model=model
184
+ )
185
+ )
186
+ else:
187
+ # For other providers, create a generic async response
188
+ invocation_arn = response_list[0].get("invocationArn", "")
189
+
190
+ from litellm.types.utils import Embedding, Usage
191
+
192
+ embedding = Embedding(
193
+ embedding=[],
194
+ index=0,
195
+ object="embedding", # Must be literal "embedding"
196
+ )
197
+ usage = Usage(prompt_tokens=0, total_tokens=0)
198
+
199
+ # Create hidden params with job ID
200
+ from litellm.types.llms.base import HiddenParams
201
+
202
+ hidden_params = HiddenParams()
203
+ setattr(hidden_params, "_invocation_arn", invocation_arn)
204
+
205
+ returned_response = EmbeddingResponse(
206
+ data=[embedding],
207
+ model=model,
208
+ usage=usage,
209
+ hidden_params=hidden_params,
210
+ )
211
+ else:
212
+ # Handle regular invoke responses
213
+ if model == "amazon.titan-embed-image-v1":
214
+ returned_response = (
215
+ AmazonTitanMultimodalEmbeddingG1Config()._transform_response(
216
+ response_list=response_list, model=model, batch_data=batch_data
217
+ )
218
+ )
219
+ elif model == "amazon.titan-embed-text-v1":
220
+ returned_response = AmazonTitanG1Config()._transform_response(
221
+ response_list=response_list, model=model
222
+ )
223
+ elif model == "amazon.titan-embed-text-v2:0":
224
+ returned_response = AmazonTitanV2Config()._transform_response(
225
+ response_list=response_list, model=model
226
+ )
227
+ elif provider == "twelvelabs":
228
+ returned_response = (
229
+ TwelveLabsMarengoEmbeddingConfig()._transform_response(
230
+ response_list=response_list, model=model
231
+ )
232
+ )
233
+ elif provider == "nova":
234
+ returned_response = AmazonNovaEmbeddingConfig()._transform_response(
235
+ response_list=response_list, model=model, batch_data=batch_data
236
+ )
237
+
238
+ ##########################################################
239
+ # Validate returned response
240
+ ##########################################################
241
+ if returned_response is None:
242
+ raise Exception(
243
+ "Unable to map model response to known provider format. model={}".format(
244
+ model
245
+ )
246
+ )
247
+ return returned_response
248
+
249
+ def _single_func_embeddings(
250
+ self,
251
+ client: Optional[HTTPHandler],
252
+ timeout: Optional[Union[float, httpx.Timeout]],
253
+ batch_data: List[dict],
254
+ credentials: Any,
255
+ extra_headers: Optional[dict],
256
+ endpoint_url: str,
257
+ aws_region_name: str,
258
+ model: str,
259
+ logging_obj: Any,
260
+ provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
261
+ api_key: Optional[str] = None,
262
+ is_async_invoke: Optional[bool] = False,
263
+ ):
264
+ responses: List[dict] = []
265
+ for data in batch_data:
266
+ headers = {"Content-Type": "application/json"}
267
+ if extra_headers is not None:
268
+ headers = {"Content-Type": "application/json", **extra_headers}
269
+
270
+ prepped = self.get_request_headers( # type: ignore # type: ignore
271
+ credentials=credentials,
272
+ aws_region_name=aws_region_name,
273
+ extra_headers=extra_headers,
274
+ endpoint_url=endpoint_url,
275
+ data=json.dumps(data),
276
+ headers=headers,
277
+ api_key=api_key,
278
+ )
279
+
280
+ ## LOGGING
281
+ logging_obj.pre_call(
282
+ input=data,
283
+ api_key="",
284
+ additional_args={
285
+ "complete_input_dict": data,
286
+ "api_base": prepped.url,
287
+ "headers": prepped.headers,
288
+ },
289
+ )
290
+ headers_for_request = (
291
+ dict(prepped.headers) if hasattr(prepped, "headers") else {}
292
+ )
293
+ response = self._make_sync_call(
294
+ client=client,
295
+ timeout=timeout,
296
+ api_base=prepped.url,
297
+ headers=headers_for_request,
298
+ data=data,
299
+ )
300
+
301
+ ## LOGGING
302
+ logging_obj.post_call(
303
+ input=data,
304
+ api_key="",
305
+ original_response=response,
306
+ additional_args={"complete_input_dict": data},
307
+ )
308
+
309
+ responses.append(response)
310
+
311
+ return self._transform_response(
312
+ response_list=responses,
313
+ model=model,
314
+ provider=provider,
315
+ is_async_invoke=is_async_invoke,
316
+ batch_data=batch_data,
317
+ )
318
+
319
+ async def _async_single_func_embeddings(
320
+ self,
321
+ client: Optional[AsyncHTTPHandler],
322
+ timeout: Optional[Union[float, httpx.Timeout]],
323
+ batch_data: List[dict],
324
+ credentials: Any,
325
+ extra_headers: Optional[dict],
326
+ endpoint_url: str,
327
+ aws_region_name: str,
328
+ model: str,
329
+ logging_obj: Any,
330
+ provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
331
+ api_key: Optional[str] = None,
332
+ is_async_invoke: Optional[bool] = False,
333
+ ):
334
+ responses: List[dict] = []
335
+ for data in batch_data:
336
+ headers = {"Content-Type": "application/json"}
337
+ if extra_headers is not None:
338
+ headers = {"Content-Type": "application/json", **extra_headers}
339
+
340
+ prepped = self.get_request_headers( # type: ignore # type: ignore
341
+ credentials=credentials,
342
+ aws_region_name=aws_region_name,
343
+ extra_headers=extra_headers,
344
+ endpoint_url=endpoint_url,
345
+ data=json.dumps(data),
346
+ headers=headers,
347
+ api_key=api_key,
348
+ )
349
+
350
+ ## LOGGING
351
+ logging_obj.pre_call(
352
+ input=data,
353
+ api_key="",
354
+ additional_args={
355
+ "complete_input_dict": data,
356
+ "api_base": prepped.url,
357
+ "headers": prepped.headers,
358
+ },
359
+ )
360
+ # Convert CaseInsensitiveDict to regular dict for httpx compatibility
361
+ # This ensures custom headers are properly forwarded, especially with IAM roles and custom api_base
362
+ headers_for_request = (
363
+ dict(prepped.headers) if hasattr(prepped, "headers") else {}
364
+ )
365
+ response = await self._make_async_call(
366
+ client=client,
367
+ timeout=timeout,
368
+ api_base=prepped.url,
369
+ headers=headers_for_request,
370
+ data=data,
371
+ )
372
+
373
+ ## LOGGING
374
+ logging_obj.post_call(
375
+ input=data,
376
+ api_key="",
377
+ original_response=response,
378
+ additional_args={"complete_input_dict": data},
379
+ )
380
+
381
+ responses.append(response)
382
+ ## TRANSFORM RESPONSE ##
383
+ return self._transform_response(
384
+ response_list=responses,
385
+ model=model,
386
+ provider=provider,
387
+ is_async_invoke=is_async_invoke,
388
+ batch_data=batch_data,
389
+ )
390
+
391
+ def embeddings( # noqa: PLR0915
392
+ self,
393
+ model: str,
394
+ input: List[str],
395
+ api_base: Optional[str],
396
+ model_response: EmbeddingResponse,
397
+ print_verbose: Callable,
398
+ encoding,
399
+ logging_obj,
400
+ client: Optional[Union[HTTPHandler, AsyncHTTPHandler]],
401
+ timeout: Optional[Union[float, httpx.Timeout]],
402
+ aembedding: Optional[bool],
403
+ extra_headers: Optional[dict],
404
+ optional_params: dict,
405
+ litellm_params: dict,
406
+ api_key: Optional[str] = None,
407
+ ) -> EmbeddingResponse:
408
+ credentials, aws_region_name = self._load_credentials(optional_params)
409
+
410
+ ### TRANSFORMATION ###
411
+ unencoded_model_id = (
412
+ optional_params.pop("model_id", None) or model
413
+ ) # default to model if not passed
414
+ modelId = urllib.parse.quote(unencoded_model_id, safe="")
415
+ aws_region_name = self._get_aws_region_name(
416
+ optional_params={"aws_region_name": aws_region_name},
417
+ model=model,
418
+ model_id=unencoded_model_id,
419
+ )
420
+ # Check async invoke needs to be used
421
+ has_async_invoke = "async_invoke/" in model
422
+ if has_async_invoke:
423
+ model = model.replace("async_invoke/", "", 1)
424
+ provider = self.get_bedrock_embedding_provider(model)
425
+ if provider is None:
426
+ raise Exception(
427
+ f"Unable to determine bedrock embedding provider for model: {model}. "
428
+ f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}"
429
+ )
430
+ inference_params = copy.deepcopy(optional_params)
431
+ inference_params = {
432
+ k: v
433
+ for k, v in inference_params.items()
434
+ if k.lower() not in self.aws_authentication_params
435
+ }
436
+ inference_params.pop(
437
+ "user", None
438
+ ) # make sure user is not passed in for bedrock call
439
+
440
+ data: Optional[CohereEmbeddingRequest] = None
441
+ batch_data: Optional[List] = None
442
+ if provider == "cohere":
443
+ data = BedrockCohereEmbeddingConfig()._transform_request(
444
+ model=model, input=input, inference_params=inference_params
445
+ )
446
+ elif provider == "amazon" and model in [
447
+ "amazon.titan-embed-image-v1",
448
+ "amazon.titan-embed-text-v1",
449
+ "amazon.titan-embed-text-v2:0",
450
+ ]:
451
+ batch_data = []
452
+ for i in input:
453
+ if model == "amazon.titan-embed-image-v1":
454
+ transformed_request: (
455
+ AmazonEmbeddingRequest
456
+ ) = AmazonTitanMultimodalEmbeddingG1Config()._transform_request(
457
+ input=i, inference_params=inference_params
458
+ )
459
+ elif model == "amazon.titan-embed-text-v1":
460
+ transformed_request = AmazonTitanG1Config()._transform_request(
461
+ input=i, inference_params=inference_params
462
+ )
463
+ elif model == "amazon.titan-embed-text-v2:0":
464
+ transformed_request = AmazonTitanV2Config()._transform_request(
465
+ input=i, inference_params=inference_params
466
+ )
467
+ else:
468
+ raise Exception(
469
+ "Unmapped model. Received={}. Expected={}".format(
470
+ model,
471
+ [
472
+ "amazon.titan-embed-image-v1",
473
+ "amazon.titan-embed-text-v1",
474
+ "amazon.titan-embed-text-v2:0",
475
+ ],
476
+ )
477
+ )
478
+ batch_data.append(transformed_request)
479
+ elif provider == "twelvelabs":
480
+ batch_data = []
481
+ for i in input:
482
+ twelvelabs_request = (
483
+ TwelveLabsMarengoEmbeddingConfig()._transform_request(
484
+ input=i,
485
+ inference_params=inference_params,
486
+ async_invoke_route=has_async_invoke,
487
+ model_id=modelId,
488
+ output_s3_uri=inference_params.get("output_s3_uri"),
489
+ )
490
+ )
491
+ batch_data.append(twelvelabs_request)
492
+ elif provider == "nova":
493
+ batch_data = []
494
+ for i in input:
495
+ nova_request = AmazonNovaEmbeddingConfig()._transform_request(
496
+ input=i,
497
+ inference_params=inference_params,
498
+ async_invoke_route=has_async_invoke,
499
+ model_id=modelId,
500
+ output_s3_uri=inference_params.get("output_s3_uri"),
501
+ )
502
+ batch_data.append(nova_request)
503
+
504
+ ### SET RUNTIME ENDPOINT ###
505
+ endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
506
+ api_base=api_base,
507
+ aws_bedrock_runtime_endpoint=optional_params.pop(
508
+ "aws_bedrock_runtime_endpoint", None
509
+ ),
510
+ aws_region_name=aws_region_name,
511
+ )
512
+ if has_async_invoke:
513
+ endpoint_url = f"{endpoint_url}/async-invoke"
514
+ else:
515
+ endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
516
+
517
+ if batch_data is not None:
518
+ if aembedding:
519
+ return self._async_single_func_embeddings( # type: ignore
520
+ client=(
521
+ client
522
+ if client is not None and isinstance(client, AsyncHTTPHandler)
523
+ else None
524
+ ),
525
+ timeout=timeout,
526
+ batch_data=batch_data,
527
+ credentials=credentials,
528
+ extra_headers=extra_headers,
529
+ endpoint_url=endpoint_url,
530
+ aws_region_name=aws_region_name,
531
+ model=model,
532
+ logging_obj=logging_obj,
533
+ api_key=api_key,
534
+ provider=provider,
535
+ is_async_invoke=has_async_invoke,
536
+ )
537
+ returned_response = self._single_func_embeddings(
538
+ client=(
539
+ client
540
+ if client is not None and isinstance(client, HTTPHandler)
541
+ else None
542
+ ),
543
+ timeout=timeout,
544
+ batch_data=batch_data,
545
+ credentials=credentials,
546
+ extra_headers=extra_headers,
547
+ endpoint_url=endpoint_url,
548
+ aws_region_name=aws_region_name,
549
+ model=model,
550
+ logging_obj=logging_obj,
551
+ api_key=api_key,
552
+ provider=provider,
553
+ is_async_invoke=has_async_invoke,
554
+ )
555
+ if returned_response is None:
556
+ raise Exception("Unable to map Bedrock request to provider")
557
+ return returned_response
558
+ elif data is None:
559
+ raise Exception("Unable to map Bedrock request to provider")
560
+
561
+ headers = {"Content-Type": "application/json"}
562
+ if extra_headers is not None:
563
+ headers = {"Content-Type": "application/json", **extra_headers}
564
+
565
+ prepped = self.get_request_headers( # type: ignore
566
+ credentials=credentials,
567
+ aws_region_name=aws_region_name,
568
+ extra_headers=extra_headers,
569
+ endpoint_url=endpoint_url,
570
+ data=json.dumps(data),
571
+ headers=headers,
572
+ api_key=api_key,
573
+ )
574
+
575
+ ## ROUTING ##
576
+ # Convert CaseInsensitiveDict to regular dict for httpx compatibility
577
+ headers_for_request = (
578
+ dict(prepped.headers) if hasattr(prepped, "headers") else {}
579
+ )
580
+ return cohere_embedding(
581
+ model=model,
582
+ input=input,
583
+ model_response=model_response,
584
+ logging_obj=logging_obj,
585
+ optional_params=optional_params,
586
+ encoding=encoding,
587
+ data=data, # type: ignore
588
+ complete_api_base=prepped.url,
589
+ api_key=None,
590
+ aembedding=aembedding,
591
+ timeout=timeout,
592
+ client=client,
593
+ headers=headers_for_request,
594
+ )
595
+
596
+ async def _get_async_invoke_status(
597
+ self, invocation_arn: str, aws_region_name: str, logging_obj=None, **kwargs
598
+ ) -> dict:
599
+ """
600
+ Get the status of an async invoke job using the GetAsyncInvoke operation.
601
+
602
+ Args:
603
+ invocation_arn: The invocation ARN from the async invoke response
604
+ aws_region_name: AWS region name
605
+ **kwargs: Additional parameters (credentials, etc.)
606
+
607
+ Returns:
608
+ dict: Status response from AWS Bedrock
609
+ """
610
+
611
+ # Get AWS credentials using the same method as other Bedrock methods
612
+ credentials, _ = self._load_credentials(kwargs)
613
+
614
+ # Get the runtime endpoint
615
+ endpoint_url, _ = self.get_runtime_endpoint(
616
+ api_base=None,
617
+ aws_bedrock_runtime_endpoint=kwargs.get("aws_bedrock_runtime_endpoint"),
618
+ aws_region_name=aws_region_name,
619
+ )
620
+
621
+ from urllib.parse import quote
622
+
623
+ # Encode the ARN for use in URL path
624
+ encoded_arn = quote(invocation_arn, safe="")
625
+ status_url = f"{endpoint_url.rstrip('/')}/async-invoke/{encoded_arn}"
626
+
627
+ # Prepare headers for GET request
628
+ headers = {"Content-Type": "application/json"}
629
+
630
+ # Use AWSRequest directly for GET requests (get_request_headers hardcodes POST)
631
+ try:
632
+ from botocore.auth import SigV4Auth
633
+ from botocore.awsrequest import AWSRequest
634
+ except ImportError:
635
+ raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
636
+
637
+ # Create AWSRequest with GET method and encoded URL
638
+ request = AWSRequest(
639
+ method="GET",
640
+ url=status_url,
641
+ data=None, # GET request, no body
642
+ headers=headers,
643
+ )
644
+
645
+ # Sign the request - SigV4Auth will create canonical string from request URL
646
+ sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name)
647
+ sigv4.add_auth(request)
648
+
649
+ # Prepare the request
650
+ prepped = request.prepare()
651
+
652
+ # LOGGING
653
+ if logging_obj is not None:
654
+ # Create custom curl command for GET request
655
+ masked_headers = logging_obj._get_masked_headers(prepped.headers)
656
+ formatted_headers = " ".join(
657
+ [f"-H '{k}: {v}'" for k, v in masked_headers.items()]
658
+ )
659
+ custom_curl = "\n\nGET Request Sent from LiteLLM:\n"
660
+ custom_curl += "curl -X GET \\\n"
661
+ custom_curl += f"{prepped.url} \\\n"
662
+ custom_curl += f"{formatted_headers}\n"
663
+
664
+ logging_obj.pre_call(
665
+ input=invocation_arn,
666
+ api_key="",
667
+ additional_args={
668
+ "complete_input_dict": {"invocation_arn": invocation_arn},
669
+ "api_base": prepped.url,
670
+ "headers": prepped.headers,
671
+ "request_str": custom_curl, # Override with custom GET curl command
672
+ },
673
+ )
674
+
675
+ # Make the GET request
676
+ client = get_async_httpx_client(llm_provider=LlmProviders.BEDROCK)
677
+ response = await client.get(
678
+ url=prepped.url,
679
+ headers=prepped.headers,
680
+ )
681
+
682
+ # LOGGING
683
+ if logging_obj is not None:
684
+ logging_obj.post_call(
685
+ input=invocation_arn,
686
+ api_key="",
687
+ original_response=response,
688
+ additional_args={
689
+ "complete_input_dict": {"invocation_arn": invocation_arn}
690
+ },
691
+ )
692
+
693
+ # Parse response
694
+ if response.status_code == 200:
695
+ return response.json()
696
+ else:
697
+ raise Exception(
698
+ f"Failed to get async invoke status: {response.status_code} - {response.text}"
699
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py ADDED
@@ -0,0 +1,304 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation logic from OpenAI /v1/embeddings format to Bedrock TwelveLabs Marengo /invoke and /async-invoke format.
3
+
4
+ Why separate file? Make it easy to see how transformation works
5
+
6
+ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html
7
+ """
8
+
9
+ from typing import List, Optional, Union, cast
10
+
11
+ from litellm.types.llms.bedrock import (
12
+ TWELVELABS_EMBEDDING_INPUT_TYPES,
13
+ TwelveLabsAsyncInvokeRequest,
14
+ TwelveLabsMarengoEmbeddingRequest,
15
+ TwelveLabsOutputDataConfig,
16
+ TwelveLabsS3Location,
17
+ TwelveLabsS3OutputDataConfig,
18
+ )
19
+ from litellm.types.utils import Embedding, EmbeddingResponse, Usage
20
+
21
+
22
+ class TwelveLabsMarengoEmbeddingConfig:
23
+ """
24
+ Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html
25
+
26
+ Supports text, image, video, and audio inputs.
27
+ - InvokeModel: text and image inputs
28
+ - StartAsyncInvoke: video, audio, image, and text inputs
29
+ """
30
+
31
+ def __init__(self) -> None:
32
+ pass
33
+
34
+ def get_supported_openai_params(self) -> List[str]:
35
+ return [
36
+ "encoding_format",
37
+ "textTruncate",
38
+ "embeddingOption",
39
+ "startSec",
40
+ "lengthSec",
41
+ "useFixedLengthSec",
42
+ "minClipSec",
43
+ "input_type",
44
+ ]
45
+
46
+ def map_openai_params(
47
+ self, non_default_params: dict, optional_params: dict
48
+ ) -> dict:
49
+ for k, v in non_default_params.items():
50
+ if k == "encoding_format":
51
+ # TwelveLabs doesn't have encoding_format, but we can map it to embeddingOption
52
+ if v == "float":
53
+ optional_params["embeddingOption"] = ["visual-text", "visual-image"]
54
+ elif k == "textTruncate":
55
+ optional_params["textTruncate"] = v
56
+ elif k == "embeddingOption":
57
+ optional_params["embeddingOption"] = v
58
+ elif k == "input_type":
59
+ # Map input_type to inputType for Bedrock
60
+ optional_params["inputType"] = v
61
+ elif k in ["startSec", "lengthSec", "useFixedLengthSec", "minClipSec"]:
62
+ optional_params[k] = v
63
+ return optional_params
64
+
65
+ def _extract_bucket_owner_from_params(self, inference_params: dict) -> str:
66
+ """
67
+ Extract bucket owner from inference parameters.
68
+ """
69
+ return inference_params.get("bucketOwner", "")
70
+
71
+ def _is_s3_url(self, input: str) -> bool:
72
+ """Check if input is an S3 URL."""
73
+ return input.startswith("s3://")
74
+
75
+ def _transform_request(
76
+ self,
77
+ input: str,
78
+ inference_params: dict,
79
+ async_invoke_route: bool = False,
80
+ model_id: Optional[str] = None,
81
+ output_s3_uri: Optional[str] = None,
82
+ ) -> Union[TwelveLabsMarengoEmbeddingRequest, TwelveLabsAsyncInvokeRequest]:
83
+ """
84
+ Transform OpenAI-style input to TwelveLabs Marengo format/async-invoke format.
85
+
86
+ Supports:
87
+ - Text inputs (for both invoke and async-invoke)
88
+ - Image inputs (for both invoke and async-invoke)
89
+ - Video inputs (async-invoke only)
90
+ - Audio inputs (async-invoke only)
91
+ - S3 URLs for all media types (async-invoke only)
92
+ """
93
+ # Get input_type or default to "text"
94
+ input_type = cast(
95
+ TWELVELABS_EMBEDDING_INPUT_TYPES,
96
+ inference_params.get("inputType")
97
+ or inference_params.get("input_type")
98
+ or "text",
99
+ )
100
+
101
+ # Validate that async-invoke is used for video/audio
102
+ if input_type in ["video", "audio"] and not async_invoke_route:
103
+ raise ValueError(
104
+ f"Input type '{input_type}' requires async_invoke route. "
105
+ f"Use model format: 'bedrock/async_invoke/model_id'"
106
+ )
107
+
108
+ transformed_request: TwelveLabsMarengoEmbeddingRequest = {
109
+ "inputType": input_type
110
+ }
111
+
112
+ if input_type == "text":
113
+ transformed_request["inputText"] = input
114
+ # Set default textTruncate if not specified
115
+ if "textTruncate" not in inference_params:
116
+ transformed_request["textTruncate"] = "end"
117
+
118
+ elif input_type in ["image", "video", "audio"]:
119
+ if self._is_s3_url(input):
120
+ # S3 URL input
121
+ s3_location: TwelveLabsS3Location = {"uri": input}
122
+ bucket_owner = self._extract_bucket_owner_from_params(inference_params)
123
+ if bucket_owner:
124
+ s3_location["bucketOwner"] = bucket_owner
125
+
126
+ transformed_request["mediaSource"] = {"s3Location": s3_location}
127
+ else:
128
+ # Base64 encoded input
129
+ if input.startswith("data:"):
130
+ # Extract base64 data from data URL
131
+ b64_str = input.split(",", 1)[1] if "," in input else input
132
+ else:
133
+ # Direct base64 string
134
+ from litellm.utils import get_base64_str
135
+
136
+ b64_str = get_base64_str(input)
137
+
138
+ transformed_request["mediaSource"] = {"base64String": b64_str}
139
+
140
+ # Apply any additional inference parameters
141
+ for k, v in inference_params.items():
142
+ if k not in [
143
+ "inputType",
144
+ "input_type", # Exclude both camelCase and snake_case
145
+ "inputText",
146
+ "mediaSource",
147
+ "bucketOwner", # Don't include bucketOwner in the request
148
+ ]: # Don't override core fields
149
+ transformed_request[k] = v # type: ignore
150
+
151
+ # If async invoke route, wrap in the async invoke format
152
+ if async_invoke_route and model_id:
153
+ return self._wrap_async_invoke_request(
154
+ model_input=transformed_request,
155
+ model_id=model_id,
156
+ output_s3_uri=output_s3_uri,
157
+ )
158
+
159
+ return transformed_request
160
+
161
+ def _wrap_async_invoke_request(
162
+ self,
163
+ model_input: TwelveLabsMarengoEmbeddingRequest,
164
+ model_id: str,
165
+ output_s3_uri: Optional[str] = None,
166
+ ) -> TwelveLabsAsyncInvokeRequest:
167
+ """
168
+ Wrap the transformed request in the correct AWS Bedrock async invoke format.
169
+
170
+ Args:
171
+ model_input: The transformed TwelveLabs Marengo embedding request
172
+ model_id: The model identifier (without async_invoke prefix)
173
+ output_s3_uri: Optional S3 URI for output data config
174
+
175
+ Returns:
176
+ TwelveLabsAsyncInvokeRequest: The wrapped async invoke request
177
+ """
178
+ import urllib.parse
179
+
180
+ # Clean the model ID
181
+ unquoted_model_id = urllib.parse.unquote(model_id)
182
+ if unquoted_model_id.startswith("async_invoke/"):
183
+ unquoted_model_id = unquoted_model_id.replace("async_invoke/", "")
184
+
185
+ # Validate that the S3 URI is not empty
186
+ if not output_s3_uri or output_s3_uri.strip() == "":
187
+ raise ValueError("output_s3_uri cannot be empty for async invoke requests")
188
+
189
+ return TwelveLabsAsyncInvokeRequest(
190
+ modelId=unquoted_model_id,
191
+ modelInput=model_input,
192
+ outputDataConfig=TwelveLabsOutputDataConfig(
193
+ s3OutputDataConfig=TwelveLabsS3OutputDataConfig(s3Uri=output_s3_uri)
194
+ ),
195
+ )
196
+
197
+ def _transform_response(
198
+ self, response_list: List[dict], model: str
199
+ ) -> EmbeddingResponse:
200
+ """
201
+ Transform TwelveLabs response to OpenAI format.
202
+ Handles the actual TwelveLabs response format: {"data": [{"embedding": [...]}]}
203
+ """
204
+ embeddings: List[Embedding] = []
205
+ total_tokens = 0
206
+
207
+ for response in response_list:
208
+ # TwelveLabs response format has a "data" field containing the embeddings
209
+ if "data" in response and isinstance(response["data"], list):
210
+ for item in response["data"]:
211
+ if "embedding" in item:
212
+ # Single embedding response
213
+ embedding = Embedding(
214
+ embedding=item["embedding"],
215
+ index=len(embeddings),
216
+ object="embedding",
217
+ )
218
+ embeddings.append(embedding)
219
+
220
+ # Estimate token count (rough approximation)
221
+ if "inputTextTokenCount" in item:
222
+ total_tokens += item["inputTextTokenCount"]
223
+ else:
224
+ # Rough estimate: 1 token per 4 characters for text, or use embedding size
225
+ total_tokens += len(item["embedding"]) // 4
226
+ elif "embedding" in response:
227
+ # Direct embedding response (fallback for other formats)
228
+ embedding = Embedding(
229
+ embedding=response["embedding"],
230
+ index=len(embeddings),
231
+ object="embedding",
232
+ )
233
+ embeddings.append(embedding)
234
+
235
+ # Estimate token count (rough approximation)
236
+ if "inputTextTokenCount" in response:
237
+ total_tokens += response["inputTextTokenCount"]
238
+ else:
239
+ # Rough estimate: 1 token per 4 characters for text
240
+ total_tokens += len(response.get("inputText", "")) // 4
241
+ elif "embeddings" in response:
242
+ # Multiple embeddings response (from video/audio)
243
+ for i, emb in enumerate(response["embeddings"]):
244
+ embedding = Embedding(
245
+ embedding=emb["embedding"],
246
+ index=len(embeddings),
247
+ object="embedding",
248
+ )
249
+ embeddings.append(embedding)
250
+ total_tokens += len(emb["embedding"]) // 4 # Rough estimate
251
+
252
+ usage = Usage(prompt_tokens=total_tokens, total_tokens=total_tokens)
253
+
254
+ return EmbeddingResponse(data=embeddings, model=model, usage=usage)
255
+
256
+ def _transform_async_invoke_response(
257
+ self, response: dict, model: str
258
+ ) -> EmbeddingResponse:
259
+ """
260
+ Transform async invoke response (invocation ARN) to OpenAI format.
261
+
262
+ AWS async invoke returns:
263
+ {
264
+ "invocationArn": "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"
265
+ }
266
+
267
+ We transform this to a job-like embedding response:
268
+ {
269
+ "object": "list",
270
+ "data": [
271
+ {
272
+ "object": "embedding_job_id:1234567890",
273
+ "embedding": [],
274
+ "index": 0
275
+ }
276
+ ],
277
+ "model": "model",
278
+ "usage": {}
279
+ }
280
+ """
281
+ invocation_arn = response.get("invocationArn", "")
282
+
283
+ # Create a placeholder embedding object for the job
284
+ embedding = Embedding(
285
+ embedding=[], # Empty embedding for async jobs
286
+ index=0,
287
+ object="embedding",
288
+ )
289
+
290
+ # Create usage object (empty for async jobs)
291
+ usage = Usage(prompt_tokens=0, total_tokens=0)
292
+
293
+ # Create hidden params with job ID
294
+ from litellm.types.llms.base import HiddenParams
295
+
296
+ hidden_params = HiddenParams()
297
+ setattr(hidden_params, "_invocation_arn", invocation_arn)
298
+
299
+ return EmbeddingResponse(
300
+ data=[embedding],
301
+ model=model,
302
+ usage=usage,
303
+ hidden_params=hidden_params,
304
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/files/handler.py ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import base64
3
+ from typing import Any, Coroutine, Optional, Tuple, Union
4
+
5
+ import httpx
6
+
7
+ from litellm import LlmProviders
8
+ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
9
+ from litellm.types.llms.openai import (
10
+ FileContentRequest,
11
+ HttpxBinaryResponseContent,
12
+ )
13
+ from litellm.types.utils import SpecialEnums
14
+
15
+ from ..base_aws_llm import BaseAWSLLM
16
+
17
+
18
+ class BedrockFilesHandler(BaseAWSLLM):
19
+ """
20
+ Handles downloading files from S3 for Bedrock batch processing.
21
+
22
+ This implementation downloads files from S3 buckets where Bedrock
23
+ stores batch output files.
24
+ """
25
+
26
+ def __init__(self):
27
+ super().__init__()
28
+ self.async_httpx_client = get_async_httpx_client(
29
+ llm_provider=LlmProviders.BEDROCK,
30
+ )
31
+
32
+ def _extract_s3_uri_from_file_id(self, file_id: str) -> str:
33
+ """
34
+ Extract S3 URI from encoded file ID.
35
+
36
+ The file ID can be in two formats:
37
+ 1. Base64-encoded unified file ID containing: llm_output_file_id,s3://bucket/path
38
+ 2. Direct S3 URI: s3://bucket/path
39
+
40
+ Args:
41
+ file_id: Encoded file ID or direct S3 URI
42
+
43
+ Returns:
44
+ S3 URI (e.g., "s3://bucket-name/path/to/file")
45
+ """
46
+ # First, try to decode if it's a base64-encoded unified file ID
47
+ try:
48
+ # Add padding if needed
49
+ padded = file_id + "=" * (-len(file_id) % 4)
50
+ decoded = base64.urlsafe_b64decode(padded).decode()
51
+
52
+ # Check if it's a unified file ID format
53
+ if decoded.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value):
54
+ # Extract llm_output_file_id from the decoded string
55
+ if "llm_output_file_id," in decoded:
56
+ s3_uri = decoded.split("llm_output_file_id,")[1].split(";")[0]
57
+ return s3_uri
58
+ except Exception:
59
+ pass
60
+
61
+ # If not base64 encoded or doesn't contain llm_output_file_id, assume it's already an S3 URI
62
+ if file_id.startswith("s3://"):
63
+ return file_id
64
+
65
+ # If it doesn't start with s3://, assume it's a direct S3 URI and add the prefix
66
+ return f"s3://{file_id}"
67
+
68
+ def _parse_s3_uri(self, s3_uri: str) -> Tuple[str, str]:
69
+ """
70
+ Parse S3 URI to extract bucket name and object key.
71
+
72
+ Args:
73
+ s3_uri: S3 URI (e.g., "s3://bucket-name/path/to/file")
74
+
75
+ Returns:
76
+ Tuple of (bucket_name, object_key)
77
+ """
78
+ if not s3_uri.startswith("s3://"):
79
+ raise ValueError(
80
+ f"Invalid S3 URI format: {s3_uri}. Expected format: s3://bucket-name/path/to/file"
81
+ )
82
+
83
+ # Remove 's3://' prefix
84
+ path = s3_uri[5:]
85
+
86
+ if "/" in path:
87
+ bucket_name, object_key = path.split("/", 1)
88
+ else:
89
+ bucket_name = path
90
+ object_key = ""
91
+
92
+ return bucket_name, object_key
93
+
94
+ async def afile_content(
95
+ self,
96
+ file_content_request: FileContentRequest,
97
+ optional_params: dict,
98
+ timeout: Union[float, httpx.Timeout],
99
+ max_retries: Optional[int],
100
+ ) -> HttpxBinaryResponseContent:
101
+ """
102
+ Download file content from S3 bucket for Bedrock files.
103
+
104
+ Args:
105
+ file_content_request: Contains file_id (encoded or S3 URI)
106
+ optional_params: Optional parameters containing AWS credentials
107
+ timeout: Request timeout
108
+ max_retries: Max retry attempts
109
+
110
+ Returns:
111
+ HttpxBinaryResponseContent: Binary content wrapped in compatible response format
112
+ """
113
+ import boto3
114
+ from botocore.credentials import Credentials
115
+
116
+ file_id = file_content_request.get("file_id")
117
+ if not file_id:
118
+ raise ValueError("file_id is required in file_content_request")
119
+
120
+ # Extract S3 URI from file ID
121
+ s3_uri = self._extract_s3_uri_from_file_id(file_id)
122
+ bucket_name, object_key = self._parse_s3_uri(s3_uri)
123
+
124
+ # Get AWS credentials
125
+ aws_region_name = self._get_aws_region_name(
126
+ optional_params=optional_params, model=""
127
+ )
128
+ credentials: Credentials = self.get_credentials(
129
+ aws_access_key_id=optional_params.get("aws_access_key_id"),
130
+ aws_secret_access_key=optional_params.get("aws_secret_access_key"),
131
+ aws_session_token=optional_params.get("aws_session_token"),
132
+ aws_region_name=aws_region_name,
133
+ aws_session_name=optional_params.get("aws_session_name"),
134
+ aws_profile_name=optional_params.get("aws_profile_name"),
135
+ aws_role_name=optional_params.get("aws_role_name"),
136
+ aws_web_identity_token=optional_params.get("aws_web_identity_token"),
137
+ aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
138
+ )
139
+
140
+ # Create S3 client
141
+ s3_client = boto3.client(
142
+ "s3",
143
+ aws_access_key_id=credentials.access_key,
144
+ aws_secret_access_key=credentials.secret_key,
145
+ aws_session_token=credentials.token,
146
+ region_name=aws_region_name,
147
+ verify=self._get_ssl_verify(),
148
+ )
149
+
150
+ # Download file from S3
151
+ try:
152
+ response = s3_client.get_object(Bucket=bucket_name, Key=object_key)
153
+ file_content = response["Body"].read()
154
+ except Exception as e:
155
+ raise ValueError(
156
+ f"Failed to download file from S3: {s3_uri}. Error: {str(e)}"
157
+ )
158
+
159
+ # Create mock HTTP response
160
+ mock_response = httpx.Response(
161
+ status_code=200,
162
+ content=file_content,
163
+ headers={"content-type": "application/octet-stream"},
164
+ request=httpx.Request(method="GET", url=s3_uri),
165
+ )
166
+
167
+ return HttpxBinaryResponseContent(response=mock_response)
168
+
169
+ def file_content(
170
+ self,
171
+ _is_async: bool,
172
+ file_content_request: FileContentRequest,
173
+ api_base: Optional[str],
174
+ optional_params: dict,
175
+ timeout: Union[float, httpx.Timeout],
176
+ max_retries: Optional[int],
177
+ ) -> Union[
178
+ HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
179
+ ]:
180
+ """
181
+ Download file content from S3 bucket for Bedrock files.
182
+ Supports both sync and async operations.
183
+
184
+ Args:
185
+ _is_async: Whether to run asynchronously
186
+ file_content_request: Contains file_id (encoded or S3 URI)
187
+ api_base: API base (unused for S3 operations)
188
+ optional_params: Optional parameters containing AWS credentials
189
+ timeout: Request timeout
190
+ max_retries: Max retry attempts
191
+
192
+ Returns:
193
+ HttpxBinaryResponseContent or Coroutine: Binary content wrapped in compatible response format
194
+ """
195
+ if _is_async:
196
+ return self.afile_content(
197
+ file_content_request=file_content_request,
198
+ optional_params=optional_params,
199
+ timeout=timeout,
200
+ max_retries=max_retries,
201
+ )
202
+ else:
203
+ return asyncio.run(
204
+ self.afile_content(
205
+ file_content_request=file_content_request,
206
+ optional_params=optional_params,
207
+ timeout=timeout,
208
+ max_retries=max_retries,
209
+ )
210
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/files/transformation.py ADDED
@@ -0,0 +1,786 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import time
4
+ from typing import Any, Dict, List, Optional, Tuple, Union
5
+
6
+ import httpx
7
+ from httpx import Headers, Response
8
+ from openai.types.file_deleted import FileDeleted
9
+
10
+ from litellm._logging import verbose_logger
11
+ from litellm._uuid import uuid
12
+ from litellm.files.utils import FilesAPIUtils
13
+ from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
14
+ from litellm.llms.base_llm.chat.transformation import BaseLLMException
15
+ from litellm.llms.base_llm.files.transformation import (
16
+ BaseFilesConfig,
17
+ LiteLLMLoggingObj,
18
+ )
19
+ from litellm.types.llms.openai import (
20
+ AllMessageValues,
21
+ CreateFileRequest,
22
+ FileTypes,
23
+ HttpxBinaryResponseContent,
24
+ OpenAICreateFileRequestOptionalParams,
25
+ OpenAIFileObject,
26
+ PathLike,
27
+ )
28
+ from litellm.types.utils import ExtractedFileData, LlmProviders
29
+ from litellm.utils import get_llm_provider
30
+
31
+ from ..base_aws_llm import BaseAWSLLM
32
+ from ..common_utils import BedrockError
33
+
34
+
35
+ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
36
+ """
37
+ Config for Bedrock Files - handles S3 uploads for Bedrock batch processing
38
+ """
39
+
40
+ def __init__(self):
41
+ self.jsonl_transformation = BedrockJsonlFilesTransformation()
42
+ super().__init__()
43
+
44
+ @property
45
+ def custom_llm_provider(self) -> LlmProviders:
46
+ return LlmProviders.BEDROCK
47
+
48
+ @property
49
+ def file_upload_http_method(self) -> str:
50
+ """
51
+ Bedrock files are uploaded to S3, which requires PUT requests
52
+ """
53
+ return "PUT"
54
+
55
+ def validate_environment(
56
+ self,
57
+ headers: dict,
58
+ model: str,
59
+ messages: List[AllMessageValues],
60
+ optional_params: dict,
61
+ litellm_params: dict,
62
+ api_key: Optional[str] = None,
63
+ api_base: Optional[str] = None,
64
+ ) -> dict:
65
+ # No additional headers needed for S3 uploads - AWS credentials handled by BaseAWSLLM
66
+ return headers
67
+
68
+ def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
69
+ """
70
+ Helper to extract content from various OpenAI file types and return as string.
71
+
72
+ Handles:
73
+ - Direct content (str, bytes, IO[bytes])
74
+ - Tuple formats: (filename, content, [content_type], [headers])
75
+ - PathLike objects
76
+ """
77
+ content: Union[str, bytes] = b""
78
+ # Extract file content from tuple if necessary
79
+ if isinstance(openai_file_content, tuple):
80
+ # Take the second element which is always the file content
81
+ file_content = openai_file_content[1]
82
+ else:
83
+ file_content = openai_file_content
84
+
85
+ # Handle different file content types
86
+ if isinstance(file_content, str):
87
+ # String content can be used directly
88
+ content = file_content
89
+ elif isinstance(file_content, bytes):
90
+ # Bytes content can be decoded
91
+ content = file_content
92
+ elif isinstance(file_content, PathLike): # PathLike
93
+ with open(str(file_content), "rb") as f:
94
+ content = f.read()
95
+ elif hasattr(file_content, "read"): # IO[bytes]
96
+ # File-like objects need to be read
97
+ content = file_content.read()
98
+
99
+ # Ensure content is string
100
+ if isinstance(content, bytes):
101
+ content = content.decode("utf-8")
102
+
103
+ return content
104
+
105
+ def _get_s3_object_name_from_batch_jsonl(
106
+ self,
107
+ openai_jsonl_content: List[Dict[str, Any]],
108
+ ) -> str:
109
+ """
110
+ Gets a unique S3 object name for the Bedrock batch processing job
111
+
112
+ named as: litellm-bedrock-files/{model}/{uuid}
113
+ """
114
+ _model = openai_jsonl_content[0].get("body", {}).get("model", "")
115
+ # Remove bedrock/ prefix if present
116
+ if _model.startswith("bedrock/"):
117
+ _model = _model[8:]
118
+
119
+ # Replace colons with hyphens for Bedrock S3 URI compliance
120
+ _model = _model.replace(":", "-")
121
+
122
+ object_name = f"litellm-bedrock-files-{_model}-{uuid.uuid4()}.jsonl"
123
+ return object_name
124
+
125
+ def get_object_name(
126
+ self, extracted_file_data: ExtractedFileData, purpose: str
127
+ ) -> str:
128
+ """
129
+ Get the object name for the request
130
+ """
131
+ extracted_file_data_content = extracted_file_data.get("content")
132
+
133
+ if extracted_file_data_content is None:
134
+ raise ValueError("file content is required")
135
+
136
+ if purpose == "batch":
137
+ ## 1. If jsonl, check if there's a model name
138
+ file_content = self._get_content_from_openai_file(
139
+ extracted_file_data_content
140
+ )
141
+
142
+ # Split into lines and parse each line as JSON
143
+ openai_jsonl_content = [
144
+ json.loads(line) for line in file_content.splitlines() if line.strip()
145
+ ]
146
+ if len(openai_jsonl_content) > 0:
147
+ return self._get_s3_object_name_from_batch_jsonl(openai_jsonl_content)
148
+
149
+ ## 2. If not jsonl, return the filename
150
+ filename = extracted_file_data.get("filename")
151
+ if filename:
152
+ return filename
153
+ ## 3. If no file name, return timestamp
154
+ return str(int(time.time()))
155
+
156
+ def get_complete_file_url(
157
+ self,
158
+ api_base: Optional[str],
159
+ api_key: Optional[str],
160
+ model: str,
161
+ optional_params: Dict,
162
+ litellm_params: Dict,
163
+ data: CreateFileRequest,
164
+ ) -> str:
165
+ """
166
+ Get the complete S3 URL for the file upload request
167
+ """
168
+ bucket_name = litellm_params.get("s3_bucket_name") or os.getenv(
169
+ "AWS_S3_BUCKET_NAME"
170
+ )
171
+ if not bucket_name:
172
+ raise ValueError(
173
+ "S3 bucket_name is required. Set 's3_bucket_name' in litellm_params or AWS_S3_BUCKET_NAME env var"
174
+ )
175
+
176
+ s3_region_name = litellm_params.get("s3_region_name") or optional_params.get(
177
+ "s3_region_name"
178
+ )
179
+ aws_region_name = s3_region_name or self._get_aws_region_name(
180
+ optional_params, model
181
+ )
182
+
183
+ file_data = data.get("file")
184
+ purpose = data.get("purpose")
185
+ if file_data is None:
186
+ raise ValueError("file is required")
187
+ if purpose is None:
188
+ raise ValueError("purpose is required")
189
+ extracted_file_data = extract_file_data(file_data)
190
+ object_name = self.get_object_name(extracted_file_data, purpose)
191
+
192
+ # S3 endpoint URL format
193
+ s3_endpoint_url = (
194
+ optional_params.get("s3_endpoint_url")
195
+ or f"https://s3.{aws_region_name}.amazonaws.com"
196
+ )
197
+
198
+ return f"{s3_endpoint_url}/{bucket_name}/{object_name}"
199
+
200
+ def get_supported_openai_params(
201
+ self, model: str
202
+ ) -> List[OpenAICreateFileRequestOptionalParams]:
203
+ return []
204
+
205
+ def map_openai_params(
206
+ self,
207
+ non_default_params: dict,
208
+ optional_params: dict,
209
+ model: str,
210
+ drop_params: bool,
211
+ ) -> dict:
212
+ return optional_params
213
+
214
+ # Providers whose InvokeModel body uses the Converse API format
215
+ # (messages + inferenceConfig + image blocks). Nova is the primary
216
+ # example; add others here as they adopt the same schema.
217
+ CONVERSE_INVOKE_PROVIDERS = ("nova",)
218
+
219
+ def _map_openai_to_bedrock_params(
220
+ self,
221
+ openai_request_body: Dict[str, Any],
222
+ provider: Optional[str] = None,
223
+ ) -> Dict[str, Any]:
224
+ """
225
+ Transform OpenAI request body to Bedrock-compatible modelInput
226
+ parameters using existing transformation logic.
227
+
228
+ Routes to the correct per-provider transformation so that the
229
+ resulting dict matches the InvokeModel body that Bedrock expects
230
+ for batch inference.
231
+ """
232
+ from litellm.types.utils import LlmProviders
233
+
234
+ _model = openai_request_body.get("model", "")
235
+ messages = openai_request_body.get("messages", [])
236
+ optional_params = {
237
+ k: v
238
+ for k, v in openai_request_body.items()
239
+ if k not in ["model", "messages"]
240
+ }
241
+
242
+ # --- Anthropic: use existing AmazonAnthropicClaudeConfig ---
243
+ if provider == LlmProviders.ANTHROPIC:
244
+ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
245
+ AmazonAnthropicClaudeConfig,
246
+ )
247
+
248
+ config = AmazonAnthropicClaudeConfig()
249
+ mapped_params = config.map_openai_params(
250
+ non_default_params={},
251
+ optional_params=optional_params,
252
+ model=_model,
253
+ drop_params=False,
254
+ )
255
+ return config.transform_request(
256
+ model=_model,
257
+ messages=messages,
258
+ optional_params=mapped_params,
259
+ litellm_params={},
260
+ headers={},
261
+ )
262
+
263
+ # --- Converse API providers (e.g. Nova): use AmazonConverseConfig
264
+ # to correctly convert image_url blocks to Bedrock image format
265
+ # and wrap inference params inside inferenceConfig. ---
266
+ if provider in self.CONVERSE_INVOKE_PROVIDERS:
267
+ from litellm.llms.bedrock.chat.converse_transformation import (
268
+ AmazonConverseConfig,
269
+ )
270
+
271
+ converse_config = AmazonConverseConfig()
272
+ mapped_params = converse_config.map_openai_params(
273
+ non_default_params=optional_params,
274
+ optional_params={},
275
+ model=_model,
276
+ drop_params=False,
277
+ )
278
+ return converse_config.transform_request(
279
+ model=_model,
280
+ messages=messages,
281
+ optional_params=mapped_params,
282
+ litellm_params={},
283
+ headers={},
284
+ )
285
+
286
+ # --- All other providers: passthrough (OpenAI-compatible models
287
+ # like openai.gpt-oss-*, qwen, deepseek, etc.) ---
288
+ return {
289
+ "messages": messages,
290
+ **optional_params,
291
+ }
292
+
293
+ def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
294
+ self, openai_jsonl_content: List[Dict[str, Any]]
295
+ ) -> List[Dict[str, Any]]:
296
+ """
297
+ Transforms OpenAI JSONL content to Bedrock batch format
298
+
299
+ Bedrock batch format: { "recordId": "alphanumeric string", "modelInput": {JSON body} }
300
+ Example:
301
+ {
302
+ "recordId": "CALL0000001",
303
+ "modelInput": {
304
+ "anthropic_version": "bedrock-2023-05-31",
305
+ "max_tokens": 1024,
306
+ "messages": [
307
+ {
308
+ "role": "user",
309
+ "content": [{"type": "text", "text": "Hello"}]
310
+ }
311
+ ]
312
+ }
313
+ }
314
+ """
315
+
316
+ bedrock_jsonl_content = []
317
+ for idx, _openai_jsonl_content in enumerate(openai_jsonl_content):
318
+ # Extract the request body from OpenAI format
319
+ openai_body = _openai_jsonl_content.get("body", {})
320
+ model = openai_body.get("model", "")
321
+
322
+ try:
323
+ model, _, _, _ = get_llm_provider(
324
+ model=model,
325
+ custom_llm_provider=None,
326
+ )
327
+ except Exception as e:
328
+ verbose_logger.exception(
329
+ f"litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - {str(e)}"
330
+ )
331
+
332
+ # Determine provider from model name
333
+ provider = self.get_bedrock_invoke_provider(model)
334
+
335
+ # Transform to Bedrock modelInput format
336
+ model_input = self._map_openai_to_bedrock_params(
337
+ openai_request_body=openai_body, provider=provider
338
+ )
339
+
340
+ # Create Bedrock batch record
341
+ record_id = _openai_jsonl_content.get(
342
+ "custom_id", f"CALL{str(idx).zfill(7)}"
343
+ )
344
+ bedrock_record = {"recordId": record_id, "modelInput": model_input}
345
+
346
+ bedrock_jsonl_content.append(bedrock_record)
347
+ return bedrock_jsonl_content
348
+
349
+ def transform_create_file_request(
350
+ self,
351
+ model: str,
352
+ create_file_data: CreateFileRequest,
353
+ optional_params: dict,
354
+ litellm_params: dict,
355
+ ) -> Union[bytes, str, dict]:
356
+ """
357
+ Transform file request and return a pre-signed request for S3.
358
+ This keeps the HTTP handler clean by doing all the signing here.
359
+ """
360
+ file_data = create_file_data.get("file")
361
+ if file_data is None:
362
+ raise ValueError("file is required")
363
+ extracted_file_data = extract_file_data(file_data)
364
+ extracted_file_data_content = extracted_file_data.get("content")
365
+
366
+ if extracted_file_data_content is None:
367
+ raise ValueError("file content is required")
368
+
369
+ # Get and transform the file content
370
+ if FilesAPIUtils.is_batch_jsonl_file(
371
+ create_file_data=create_file_data,
372
+ extracted_file_data=extracted_file_data,
373
+ ):
374
+ ## Transform JSONL content to Bedrock format
375
+ original_file_content = self._get_content_from_openai_file(
376
+ extracted_file_data_content
377
+ )
378
+ openai_jsonl_content = [
379
+ json.loads(line)
380
+ for line in original_file_content.splitlines()
381
+ if line.strip()
382
+ ]
383
+ bedrock_jsonl_content = (
384
+ self._transform_openai_jsonl_content_to_bedrock_jsonl_content(
385
+ openai_jsonl_content
386
+ )
387
+ )
388
+ file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
389
+ elif isinstance(extracted_file_data_content, bytes):
390
+ file_content = extracted_file_data_content.decode("utf-8")
391
+ elif isinstance(extracted_file_data_content, str):
392
+ file_content = extracted_file_data_content
393
+ else:
394
+ raise ValueError("Unsupported file content type")
395
+
396
+ # Get the S3 URL for upload
397
+ api_base = self.get_complete_file_url(
398
+ api_base=None,
399
+ api_key=None,
400
+ model=model,
401
+ optional_params=optional_params,
402
+ litellm_params=litellm_params,
403
+ data=create_file_data,
404
+ )
405
+
406
+ # s3_region_name always wins for S3 operations (same priority as in
407
+ # get_complete_file_url above). Overwrite aws_region_name unconditionally
408
+ # so the SigV4 region matches the URL region, avoiding SignatureDoesNotMatch.
409
+ s3_region_name = litellm_params.get("s3_region_name") or optional_params.get(
410
+ "s3_region_name"
411
+ )
412
+ if s3_region_name:
413
+ optional_params = {**optional_params, "aws_region_name": s3_region_name}
414
+
415
+ # Sign the request and return a pre-signed request object
416
+ signed_headers, signed_body = self._sign_s3_request(
417
+ content=file_content,
418
+ api_base=api_base,
419
+ optional_params=optional_params,
420
+ )
421
+
422
+ litellm_params["upload_url"] = api_base
423
+
424
+ # Return a dict that tells the HTTP handler exactly what to do
425
+ return {
426
+ "method": "PUT",
427
+ "url": api_base,
428
+ "headers": signed_headers,
429
+ "data": signed_body or file_content,
430
+ }
431
+
432
+ def _sign_s3_request(
433
+ self,
434
+ content: str,
435
+ api_base: str,
436
+ optional_params: dict,
437
+ ) -> Tuple[dict, str]:
438
+ """
439
+ Sign S3 PUT request using the same proven logic as S3Logger.
440
+ Reuses the exact pattern from litellm/integrations/s3_v2.py
441
+ """
442
+ try:
443
+ import hashlib
444
+
445
+ import requests
446
+ from botocore.auth import SigV4Auth
447
+ from botocore.awsrequest import AWSRequest
448
+ except ImportError:
449
+ raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
450
+
451
+ # Get AWS credentials using existing methods
452
+ aws_region_name = self._get_aws_region_name(
453
+ optional_params=optional_params, model=""
454
+ )
455
+ credentials = self.get_credentials(
456
+ aws_access_key_id=optional_params.get("aws_access_key_id"),
457
+ aws_secret_access_key=optional_params.get("aws_secret_access_key"),
458
+ aws_session_token=optional_params.get("aws_session_token"),
459
+ aws_region_name=aws_region_name,
460
+ aws_session_name=optional_params.get("aws_session_name"),
461
+ aws_profile_name=optional_params.get("aws_profile_name"),
462
+ aws_role_name=optional_params.get("aws_role_name"),
463
+ aws_web_identity_token=optional_params.get("aws_web_identity_token"),
464
+ aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
465
+ )
466
+
467
+ # Calculate SHA256 hash of the content (REQUIRED for S3)
468
+ content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
469
+
470
+ # Prepare headers with required S3 headers (same as s3_v2.py)
471
+ request_headers = {
472
+ "Content-Type": "application/json", # JSONL files are JSON content
473
+ "x-amz-content-sha256": content_hash, # REQUIRED by S3
474
+ "Content-Language": "en",
475
+ "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
476
+ }
477
+
478
+ # Use requests.Request to prepare the request (same pattern as s3_v2.py)
479
+ req = requests.Request("PUT", api_base, data=content, headers=request_headers)
480
+ prepped = req.prepare()
481
+
482
+ # Sign the request with S3 service
483
+ aws_request = AWSRequest(
484
+ method=prepped.method,
485
+ url=prepped.url,
486
+ data=prepped.body,
487
+ headers=prepped.headers,
488
+ )
489
+
490
+ # Get region name for non-LLM API calls (same as s3_v2.py)
491
+ signing_region = self.get_aws_region_name_for_non_llm_api_calls(
492
+ aws_region_name=aws_region_name
493
+ )
494
+
495
+ SigV4Auth(credentials, "s3", signing_region).add_auth(aws_request)
496
+
497
+ # Return signed headers and body
498
+ signed_body = aws_request.body
499
+ if isinstance(signed_body, bytes):
500
+ signed_body = signed_body.decode("utf-8")
501
+ elif signed_body is None:
502
+ signed_body = content # Fallback to original content
503
+
504
+ return dict(aws_request.headers), signed_body
505
+
506
+ def _convert_https_url_to_s3_uri(self, https_url: str) -> tuple[str, str]:
507
+ """
508
+ Convert HTTPS S3 URL to s3:// URI format.
509
+
510
+ Args:
511
+ https_url: HTTPS S3 URL (e.g., "https://s3.us-west-2.amazonaws.com/bucket/key")
512
+
513
+ Returns:
514
+ Tuple of (s3_uri, filename)
515
+
516
+ Example:
517
+ Input: "https://s3.us-west-2.amazonaws.com/litellm-proxy/file.jsonl"
518
+ Output: ("s3://litellm-proxy/file.jsonl", "file.jsonl")
519
+ """
520
+ import re
521
+
522
+ # Match HTTPS S3 URL patterns
523
+ # Pattern 1: https://s3.region.amazonaws.com/bucket/key
524
+ # Pattern 2: https://bucket.s3.region.amazonaws.com/key
525
+
526
+ pattern1 = r"https://s3\.([^.]+)\.amazonaws\.com/([^/]+)/(.+)"
527
+ pattern2 = r"https://([^.]+)\.s3\.([^.]+)\.amazonaws\.com/(.+)"
528
+
529
+ match1 = re.match(pattern1, https_url)
530
+ match2 = re.match(pattern2, https_url)
531
+
532
+ if match1:
533
+ # Pattern: https://s3.region.amazonaws.com/bucket/key
534
+ region, bucket, key = match1.groups()
535
+ s3_uri = f"s3://{bucket}/{key}"
536
+ elif match2:
537
+ # Pattern: https://bucket.s3.region.amazonaws.com/key
538
+ bucket, region, key = match2.groups()
539
+ s3_uri = f"s3://{bucket}/{key}"
540
+ else:
541
+ # Fallback: try to extract bucket and key from URL path
542
+ from urllib.parse import urlparse
543
+
544
+ parsed = urlparse(https_url)
545
+ path_parts = parsed.path.lstrip("/").split("/", 1)
546
+ if len(path_parts) >= 2:
547
+ bucket, key = path_parts[0], path_parts[1]
548
+ s3_uri = f"s3://{bucket}/{key}"
549
+ else:
550
+ raise ValueError(f"Unable to parse S3 URL: {https_url}")
551
+
552
+ # Extract filename from key
553
+ filename = key.split("/")[-1] if "/" in key else key
554
+
555
+ return s3_uri, filename
556
+
557
+ def transform_create_file_response(
558
+ self,
559
+ model: Optional[str],
560
+ raw_response: Response,
561
+ logging_obj: LiteLLMLoggingObj,
562
+ litellm_params: dict,
563
+ ) -> OpenAIFileObject:
564
+ """
565
+ Transform S3 File upload response into OpenAI-style FileObject
566
+ """
567
+ # For S3 uploads, we typically get an ETag and other metadata
568
+ response_headers = raw_response.headers
569
+ # Extract S3 object information from the response
570
+ # S3 PUT object returns ETag and other metadata in headers
571
+ content_length = response_headers.get("Content-Length", "0")
572
+
573
+ # Use the actual upload URL that was used for the S3 upload
574
+ upload_url = litellm_params.get("upload_url")
575
+ file_id: str = ""
576
+ filename: str = ""
577
+ if upload_url:
578
+ # Convert HTTPS S3 URL to s3:// URI format
579
+ file_id, filename = self._convert_https_url_to_s3_uri(upload_url)
580
+
581
+ return OpenAIFileObject(
582
+ purpose="batch", # Default purpose for Bedrock files
583
+ id=file_id,
584
+ filename=filename,
585
+ created_at=int(time.time()), # Current timestamp
586
+ status="uploaded",
587
+ bytes=int(content_length) if content_length.isdigit() else 0,
588
+ object="file",
589
+ )
590
+
591
+ def get_error_class(
592
+ self, error_message: str, status_code: int, headers: Union[Dict, Headers]
593
+ ) -> BaseLLMException:
594
+ return BedrockError(
595
+ status_code=status_code, message=error_message, headers=headers
596
+ )
597
+
598
+ def transform_retrieve_file_request(
599
+ self,
600
+ file_id: str,
601
+ optional_params: dict,
602
+ litellm_params: dict,
603
+ ) -> tuple[str, dict]:
604
+ raise NotImplementedError("BedrockFilesConfig does not support file retrieval")
605
+
606
+ def transform_retrieve_file_response(
607
+ self,
608
+ raw_response: httpx.Response,
609
+ logging_obj: LiteLLMLoggingObj,
610
+ litellm_params: dict,
611
+ ) -> OpenAIFileObject:
612
+ raise NotImplementedError("BedrockFilesConfig does not support file retrieval")
613
+
614
+ def transform_delete_file_request(
615
+ self,
616
+ file_id: str,
617
+ optional_params: dict,
618
+ litellm_params: dict,
619
+ ) -> tuple[str, dict]:
620
+ raise NotImplementedError("BedrockFilesConfig does not support file deletion")
621
+
622
+ def transform_delete_file_response(
623
+ self,
624
+ raw_response: httpx.Response,
625
+ logging_obj: LiteLLMLoggingObj,
626
+ litellm_params: dict,
627
+ ) -> FileDeleted:
628
+ raise NotImplementedError("BedrockFilesConfig does not support file deletion")
629
+
630
+ def transform_list_files_request(
631
+ self,
632
+ purpose: Optional[str],
633
+ optional_params: dict,
634
+ litellm_params: dict,
635
+ ) -> tuple[str, dict]:
636
+ raise NotImplementedError("BedrockFilesConfig does not support file listing")
637
+
638
+ def transform_list_files_response(
639
+ self,
640
+ raw_response: httpx.Response,
641
+ logging_obj: LiteLLMLoggingObj,
642
+ litellm_params: dict,
643
+ ) -> List[OpenAIFileObject]:
644
+ raise NotImplementedError("BedrockFilesConfig does not support file listing")
645
+
646
+ def transform_file_content_request(
647
+ self,
648
+ file_content_request,
649
+ optional_params: dict,
650
+ litellm_params: dict,
651
+ ) -> tuple[str, dict]:
652
+ raise NotImplementedError(
653
+ "BedrockFilesConfig does not support file content retrieval"
654
+ )
655
+
656
+ def transform_file_content_response(
657
+ self,
658
+ raw_response: httpx.Response,
659
+ logging_obj: LiteLLMLoggingObj,
660
+ litellm_params: dict,
661
+ ) -> HttpxBinaryResponseContent:
662
+ raise NotImplementedError(
663
+ "BedrockFilesConfig does not support file content retrieval"
664
+ )
665
+
666
+
667
+ class BedrockJsonlFilesTransformation:
668
+ """
669
+ Transforms OpenAI /v1/files/* requests to Bedrock S3 file uploads for batch processing
670
+ """
671
+
672
+ def transform_openai_file_content_to_bedrock_file_content(
673
+ self, openai_file_content: Optional[FileTypes] = None
674
+ ) -> Tuple[str, str]:
675
+ """
676
+ Transforms OpenAI FileContentRequest to Bedrock S3 file format
677
+ """
678
+
679
+ if openai_file_content is None:
680
+ raise ValueError("contents of file are None")
681
+ # Read the content of the file
682
+ file_content = self._get_content_from_openai_file(openai_file_content)
683
+
684
+ # Split into lines and parse each line as JSON
685
+ openai_jsonl_content = [
686
+ json.loads(line) for line in file_content.splitlines() if line.strip()
687
+ ]
688
+ bedrock_jsonl_content = (
689
+ self._transform_openai_jsonl_content_to_bedrock_jsonl_content(
690
+ openai_jsonl_content
691
+ )
692
+ )
693
+ bedrock_jsonl_string = "\n".join(
694
+ json.dumps(item) for item in bedrock_jsonl_content
695
+ )
696
+ object_name = self._get_s3_object_name(
697
+ openai_jsonl_content=openai_jsonl_content
698
+ )
699
+ return bedrock_jsonl_string, object_name
700
+
701
+ def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
702
+ self, openai_jsonl_content: List[Dict[str, Any]]
703
+ ):
704
+ """
705
+ Delegate to the main BedrockFilesConfig transformation method
706
+ """
707
+ config = BedrockFilesConfig()
708
+ return config._transform_openai_jsonl_content_to_bedrock_jsonl_content(
709
+ openai_jsonl_content
710
+ )
711
+
712
+ def _get_s3_object_name(
713
+ self,
714
+ openai_jsonl_content: List[Dict[str, Any]],
715
+ ) -> str:
716
+ """
717
+ Gets a unique S3 object name for the Bedrock batch processing job
718
+
719
+ named as: litellm-bedrock-files-{model}-{uuid}
720
+ """
721
+ _model = openai_jsonl_content[0].get("body", {}).get("model", "")
722
+ # Remove bedrock/ prefix if present
723
+ if _model.startswith("bedrock/"):
724
+ _model = _model[8:]
725
+ object_name = f"litellm-bedrock-files-{_model}-{uuid.uuid4()}.jsonl"
726
+ return object_name
727
+
728
+ def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
729
+ """
730
+ Helper to extract content from various OpenAI file types and return as string.
731
+
732
+ Handles:
733
+ - Direct content (str, bytes, IO[bytes])
734
+ - Tuple formats: (filename, content, [content_type], [headers])
735
+ - PathLike objects
736
+ """
737
+ content: Union[str, bytes] = b""
738
+ # Extract file content from tuple if necessary
739
+ if isinstance(openai_file_content, tuple):
740
+ # Take the second element which is always the file content
741
+ file_content = openai_file_content[1]
742
+ else:
743
+ file_content = openai_file_content
744
+
745
+ # Handle different file content types
746
+ if isinstance(file_content, str):
747
+ # String content can be used directly
748
+ content = file_content
749
+ elif isinstance(file_content, bytes):
750
+ # Bytes content can be decoded
751
+ content = file_content
752
+ elif isinstance(file_content, PathLike): # PathLike
753
+ with open(str(file_content), "rb") as f:
754
+ content = f.read()
755
+ elif hasattr(file_content, "read"): # IO[bytes]
756
+ # File-like objects need to be read
757
+ content = file_content.read()
758
+
759
+ # Ensure content is string
760
+ if isinstance(content, bytes):
761
+ content = content.decode("utf-8")
762
+
763
+ return content
764
+
765
+ def transform_s3_bucket_response_to_openai_file_object(
766
+ self, create_file_data: CreateFileRequest, s3_upload_response: Dict[str, Any]
767
+ ) -> OpenAIFileObject:
768
+ """
769
+ Transforms S3 Bucket upload file response to OpenAI FileObject
770
+ """
771
+ # S3 response typically contains ETag, key, etc.
772
+ object_key = s3_upload_response.get("Key", "")
773
+ bucket_name = s3_upload_response.get("Bucket", "")
774
+
775
+ # Extract filename from object key
776
+ filename = object_key.split("/")[-1] if "/" in object_key else object_key
777
+
778
+ return OpenAIFileObject(
779
+ purpose=create_file_data.get("purpose", "batch"),
780
+ id=f"s3://{bucket_name}/{object_key}",
781
+ filename=filename,
782
+ created_at=int(time.time()), # Current timestamp
783
+ status="uploaded",
784
+ bytes=s3_upload_response.get("ContentLength", 0),
785
+ object="file",
786
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/__init__.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Bedrock Image Edit Module
3
+
4
+ Handles image edit operations for Bedrock stability models.
5
+ """
6
+
7
+ from .handler import BedrockImageEdit
8
+
9
+ __all__ = ["BedrockImageEdit"]
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/handler.py ADDED
@@ -0,0 +1,309 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Bedrock Image Edit Handler
3
+
4
+ Handles image edit requests for Bedrock stability models.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ from typing import TYPE_CHECKING, Any, Optional, Union
11
+
12
+ import httpx
13
+ from pydantic import BaseModel
14
+
15
+ import litellm
16
+ from litellm._logging import verbose_logger
17
+ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
18
+ from litellm.llms.bedrock.image_edit.stability_transformation import (
19
+ BedrockStabilityImageEditConfig,
20
+ )
21
+ from litellm.llms.custom_httpx.http_handler import (
22
+ AsyncHTTPHandler,
23
+ HTTPHandler,
24
+ _get_httpx_client,
25
+ get_async_httpx_client,
26
+ )
27
+ from litellm.types.utils import ImageResponse
28
+
29
+ from ..base_aws_llm import BaseAWSLLM
30
+ from ..common_utils import BedrockError
31
+
32
+ if TYPE_CHECKING:
33
+ from botocore.awsrequest import AWSPreparedRequest
34
+ else:
35
+ AWSPreparedRequest = Any
36
+
37
+
38
+ class BedrockImageEditPreparedRequest(BaseModel):
39
+ """
40
+ Internal/Helper class for preparing the request for bedrock image edit
41
+ """
42
+
43
+ endpoint_url: str
44
+ prepped: AWSPreparedRequest
45
+ body: bytes
46
+ data: dict
47
+
48
+
49
+ class BedrockImageEdit(BaseAWSLLM):
50
+ """
51
+ Bedrock Image Edit handler
52
+ """
53
+
54
+ @classmethod
55
+ def get_config_class(cls, model: str | None):
56
+ if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
57
+ return BedrockStabilityImageEditConfig
58
+ else:
59
+ raise ValueError(f"Unsupported model for bedrock image edit: {model}")
60
+
61
+ def image_edit(
62
+ self,
63
+ model: str,
64
+ image: list,
65
+ prompt: Optional[str],
66
+ model_response: ImageResponse,
67
+ optional_params: dict,
68
+ logging_obj: LitellmLogging,
69
+ timeout: Optional[Union[float, httpx.Timeout]],
70
+ aimage_edit: bool = False,
71
+ api_base: Optional[str] = None,
72
+ extra_headers: Optional[dict] = None,
73
+ client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
74
+ api_key: Optional[str] = None,
75
+ ):
76
+ prepared_request = self._prepare_request(
77
+ model=model,
78
+ image=image,
79
+ prompt=prompt,
80
+ optional_params=optional_params,
81
+ api_base=api_base,
82
+ extra_headers=extra_headers,
83
+ logging_obj=logging_obj,
84
+ api_key=api_key,
85
+ )
86
+
87
+ if aimage_edit is True:
88
+ return self.async_image_edit(
89
+ prepared_request=prepared_request,
90
+ timeout=timeout,
91
+ model=model,
92
+ logging_obj=logging_obj,
93
+ prompt=prompt,
94
+ model_response=model_response,
95
+ client=(
96
+ client
97
+ if client is not None and isinstance(client, AsyncHTTPHandler)
98
+ else None
99
+ ),
100
+ )
101
+
102
+ if client is None or not isinstance(client, HTTPHandler):
103
+ client = _get_httpx_client()
104
+ try:
105
+ response = client.post(url=prepared_request.endpoint_url, headers=prepared_request.prepped.headers, data=prepared_request.body) # type: ignore
106
+ response.raise_for_status()
107
+ except httpx.HTTPStatusError as err:
108
+ error_code = err.response.status_code
109
+ raise BedrockError(status_code=error_code, message=err.response.text)
110
+ except httpx.TimeoutException:
111
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
112
+
113
+ ### FORMAT RESPONSE TO OPENAI FORMAT ###
114
+ model_response = self._transform_response_dict_to_openai_response(
115
+ model_response=model_response,
116
+ model=model,
117
+ logging_obj=logging_obj,
118
+ prompt=prompt,
119
+ response=response,
120
+ data=prepared_request.data,
121
+ )
122
+ return model_response
123
+
124
+ async def async_image_edit(
125
+ self,
126
+ prepared_request: BedrockImageEditPreparedRequest,
127
+ timeout: Optional[Union[float, httpx.Timeout]],
128
+ model: str,
129
+ logging_obj: LitellmLogging,
130
+ prompt: Optional[str],
131
+ model_response: ImageResponse,
132
+ client: Optional[AsyncHTTPHandler] = None,
133
+ ) -> ImageResponse:
134
+ """
135
+ Asynchronous handler for bedrock image edit
136
+ """
137
+ async_client = client or get_async_httpx_client(
138
+ llm_provider=litellm.LlmProviders.BEDROCK,
139
+ params={"timeout": timeout},
140
+ )
141
+
142
+ try:
143
+ response = await async_client.post(url=prepared_request.endpoint_url, headers=prepared_request.prepped.headers, data=prepared_request.body) # type: ignore
144
+ response.raise_for_status()
145
+ except httpx.HTTPStatusError as err:
146
+ error_code = err.response.status_code
147
+ raise BedrockError(status_code=error_code, message=err.response.text)
148
+ except httpx.TimeoutException:
149
+ raise BedrockError(status_code=408, message="Timeout error occurred.")
150
+
151
+ ### FORMAT RESPONSE TO OPENAI FORMAT ###
152
+ model_response = self._transform_response_dict_to_openai_response(
153
+ model=model,
154
+ logging_obj=logging_obj,
155
+ prompt=prompt,
156
+ response=response,
157
+ data=prepared_request.data,
158
+ model_response=model_response,
159
+ )
160
+ return model_response
161
+
162
+ def _prepare_request(
163
+ self,
164
+ model: str,
165
+ image: list,
166
+ prompt: Optional[str],
167
+ optional_params: dict,
168
+ api_base: Optional[str],
169
+ extra_headers: Optional[dict],
170
+ logging_obj: LitellmLogging,
171
+ api_key: Optional[str],
172
+ ) -> BedrockImageEditPreparedRequest:
173
+ """
174
+ Prepare the request body, headers, and endpoint URL for the Bedrock Image Edit API
175
+
176
+ Args:
177
+ model (str): The model to use for the image edit
178
+ image (list): The images to edit
179
+ prompt (Optional[str]): The prompt for the edit
180
+ optional_params (dict): The optional parameters for the image edit
181
+ api_base (Optional[str]): The base URL for the Bedrock API
182
+ extra_headers (Optional[dict]): The extra headers to include in the request
183
+ logging_obj (LitellmLogging): The logging object to use for logging
184
+ api_key (Optional[str]): The API key to use
185
+
186
+ Returns:
187
+ BedrockImageEditPreparedRequest: The prepared request object
188
+ """
189
+ boto3_credentials_info = self._get_boto_credentials_from_optional_params(
190
+ optional_params, model
191
+ )
192
+
193
+ # Use the existing ARN-aware provider detection method
194
+ bedrock_provider = self.get_bedrock_invoke_provider(model)
195
+ ### SET RUNTIME ENDPOINT ###
196
+ modelId = self.get_bedrock_model_id(
197
+ model=model,
198
+ provider=bedrock_provider,
199
+ optional_params=optional_params,
200
+ )
201
+ _, proxy_endpoint_url = self.get_runtime_endpoint(
202
+ api_base=api_base,
203
+ aws_bedrock_runtime_endpoint=boto3_credentials_info.aws_bedrock_runtime_endpoint,
204
+ aws_region_name=boto3_credentials_info.aws_region_name,
205
+ )
206
+ proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
207
+ data = self._get_request_body(
208
+ model=model,
209
+ image=image,
210
+ prompt=prompt,
211
+ optional_params=optional_params,
212
+ )
213
+
214
+ # Make POST Request
215
+ body = json.dumps(data).encode("utf-8")
216
+ headers = {"Content-Type": "application/json"}
217
+ if extra_headers is not None:
218
+ headers = {"Content-Type": "application/json", **extra_headers}
219
+
220
+ prepped = self.get_request_headers(
221
+ credentials=boto3_credentials_info.credentials,
222
+ aws_region_name=boto3_credentials_info.aws_region_name,
223
+ extra_headers=extra_headers,
224
+ endpoint_url=proxy_endpoint_url,
225
+ data=body,
226
+ headers=headers,
227
+ api_key=api_key,
228
+ )
229
+
230
+ ## LOGGING
231
+ logging_obj.pre_call(
232
+ input=prompt,
233
+ api_key="",
234
+ additional_args={
235
+ "complete_input_dict": data,
236
+ "api_base": proxy_endpoint_url,
237
+ "headers": prepped.headers,
238
+ },
239
+ )
240
+ return BedrockImageEditPreparedRequest(
241
+ endpoint_url=proxy_endpoint_url,
242
+ prepped=prepped,
243
+ body=body,
244
+ data=data,
245
+ )
246
+
247
+ def _get_request_body(
248
+ self,
249
+ model: str,
250
+ image: list,
251
+ prompt: Optional[str],
252
+ optional_params: dict,
253
+ ) -> dict:
254
+ """
255
+ Get the request body for the Bedrock Image Edit API
256
+
257
+ Checks the model/provider and transforms the request body accordingly
258
+
259
+ Returns:
260
+ dict: The request body to use for the Bedrock Image Edit API
261
+ """
262
+ config_class = self.get_config_class(model=model)
263
+ config_instance = config_class()
264
+ request_body, _ = config_instance.transform_image_edit_request(
265
+ model=model,
266
+ prompt=prompt,
267
+ image=image[0] if image else None,
268
+ image_edit_optional_request_params=optional_params,
269
+ litellm_params={},
270
+ headers={},
271
+ )
272
+ return dict(request_body)
273
+
274
+ def _transform_response_dict_to_openai_response(
275
+ self,
276
+ model_response: ImageResponse,
277
+ model: str,
278
+ logging_obj: LitellmLogging,
279
+ prompt: Optional[str],
280
+ response: httpx.Response,
281
+ data: dict,
282
+ ) -> ImageResponse:
283
+ """
284
+ Transforms the Image Edit response from Bedrock to OpenAI format
285
+ """
286
+
287
+ ## LOGGING
288
+ if logging_obj is not None:
289
+ logging_obj.post_call(
290
+ input=prompt,
291
+ api_key="",
292
+ original_response=response.text,
293
+ additional_args={"complete_input_dict": data},
294
+ )
295
+ verbose_logger.debug("raw model_response: %s", response.text)
296
+ response_dict = response.json()
297
+ if response_dict is None:
298
+ raise ValueError("Error in response object format, got None")
299
+
300
+ config_class = self.get_config_class(model=model)
301
+ config_instance = config_class()
302
+
303
+ model_response = config_instance.transform_image_edit_response(
304
+ model=model,
305
+ raw_response=response,
306
+ logging_obj=logging_obj,
307
+ )
308
+
309
+ return model_response
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_edit/stability_transformation.py ADDED
@@ -0,0 +1,399 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Bedrock Stability AI Image Edit Transformation
3
+
4
+ Handles transformation between OpenAI-compatible format and Bedrock Stability AI Image Edit API format.
5
+
6
+ Supported models:
7
+ - stability.stable-conservative-upscale-v1:0
8
+ - stability.stable-creative-upscale-v1:0
9
+ - stability.stable-fast-upscale-v1:0
10
+ - stability.stable-outpaint-v1:0
11
+ - stability.stable-image-control-sketch-v1:0
12
+ - stability.stable-image-control-structure-v1:0
13
+ - stability.stable-image-erase-object-v1:0
14
+ - stability.stable-image-inpaint-v1:0
15
+ - stability.stable-image-remove-background-v1:0
16
+ - stability.stable-image-search-recolor-v1:0
17
+ - stability.stable-image-search-replace-v1:0
18
+ - stability.stable-image-style-guide-v1:0
19
+ - stability.stable-style-transfer-v1:0
20
+
21
+ API Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html
22
+ """
23
+
24
+ import base64
25
+ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
26
+
27
+ import httpx
28
+
29
+ from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
30
+ from litellm.types.images.main import ImageEditOptionalRequestParams
31
+ from litellm.types.llms.stability import (
32
+ OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO,
33
+ )
34
+ from litellm.types.router import GenericLiteLLMParams
35
+ from litellm.types.utils import FileTypes, ImageObject, ImageResponse
36
+ from litellm.utils import get_model_info
37
+
38
+ if TYPE_CHECKING:
39
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
40
+
41
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
42
+ else:
43
+ LiteLLMLoggingObj = Any
44
+
45
+
46
+ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
47
+ """
48
+ Configuration for Bedrock Stability AI image edit.
49
+
50
+ Supports all Stability image edit operations through Bedrock.
51
+ """
52
+
53
+ @classmethod
54
+ def _is_stability_edit_model(cls, model: Optional[str] = None) -> bool:
55
+ """
56
+ Returns True if the model is a Bedrock Stability edit model.
57
+
58
+ Bedrock Stability edit models follow this pattern:
59
+ stability.stable-conservative-upscale-v1:0
60
+ stability.stable-creative-upscale-v1:0
61
+ stability.stable-fast-upscale-v1:0
62
+ stability.stable-outpaint-v1:0
63
+ stability.stable-image-inpaint-v1:0
64
+ stability.stable-image-erase-object-v1:0
65
+ etc.
66
+ """
67
+ if model:
68
+ model_lower = model.lower()
69
+ if "stability." in model_lower and any(
70
+ [
71
+ "upscale" in model_lower,
72
+ "outpaint" in model_lower,
73
+ "inpaint" in model_lower,
74
+ "erase" in model_lower,
75
+ "remove-background" in model_lower,
76
+ "search-recolor" in model_lower,
77
+ "search-replace" in model_lower,
78
+ "control-sketch" in model_lower,
79
+ "control-structure" in model_lower,
80
+ "style-guide" in model_lower,
81
+ "style-transfer" in model_lower,
82
+ ]
83
+ ):
84
+ return True
85
+ return False
86
+
87
+ def get_supported_openai_params(self, model: str) -> list:
88
+ """
89
+ Return list of OpenAI params supported by Bedrock Stability.
90
+ """
91
+ return [
92
+ "n", # Number of images (Stability always returns 1, we can loop)
93
+ "size", # Maps to aspect_ratio
94
+ "response_format", # b64_json or url (Stability only returns b64)
95
+ "mask",
96
+ ]
97
+
98
+ def map_openai_params(
99
+ self,
100
+ image_edit_optional_params: ImageEditOptionalRequestParams,
101
+ model: str,
102
+ drop_params: bool,
103
+ ) -> Dict:
104
+ """
105
+ Map OpenAI parameters to Bedrock Stability parameters.
106
+
107
+ OpenAI -> Stability mappings:
108
+ - size -> aspect_ratio
109
+ - n -> (handled separately, Stability returns 1 image per request)
110
+ """
111
+ supported_params = self.get_supported_openai_params(model)
112
+ # Define mapping from OpenAI params to Stability params
113
+ param_mapping = {
114
+ "size": "aspect_ratio",
115
+ # "n" and "response_format" are handled separately
116
+ }
117
+
118
+ # Create a copy to not mutate original - convert TypedDict to regular dict
119
+ mapped_params: Dict[str, Any] = dict(image_edit_optional_params)
120
+
121
+ for k, v in image_edit_optional_params.items():
122
+ if k in param_mapping:
123
+ # Map param if mapping exists and value is valid
124
+ if k == "size" and v in OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO:
125
+ mapped_params[param_mapping[k]] = OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO[v] # type: ignore
126
+ # Don't copy "size" itself to final dict
127
+ elif k == "n":
128
+ # Store for logic but do not add to outgoing params
129
+ mapped_params["_n"] = v
130
+ elif k == "response_format":
131
+ # Only b64 supported at Stability; store for postprocessing
132
+ mapped_params["_response_format"] = v
133
+ elif k not in supported_params:
134
+ if not drop_params:
135
+ raise ValueError(
136
+ f"Parameter {k} is not supported for model {model}. "
137
+ f"Supported parameters are {supported_params}. "
138
+ f"Set drop_params=True to drop unsupported parameters."
139
+ )
140
+ # Otherwise, param will simply be dropped
141
+ else:
142
+ # param is supported and not mapped, keep as-is
143
+ continue
144
+
145
+ # Remove OpenAI params that have been mapped unless they're in stability
146
+ for mapped in ["size", "n", "response_format"]:
147
+ if mapped in mapped_params:
148
+ del mapped_params[mapped]
149
+
150
+ return mapped_params
151
+
152
+ def transform_image_edit_request( # noqa: PLR0915
153
+ self,
154
+ model: str,
155
+ prompt: Optional[str],
156
+ image: Optional[FileTypes],
157
+ image_edit_optional_request_params: Dict,
158
+ litellm_params: GenericLiteLLMParams,
159
+ headers: dict,
160
+ ) -> Tuple[Dict, Any]:
161
+ """
162
+ Transform OpenAI-style request to Bedrock Stability request format.
163
+
164
+ Returns the request body dict that will be JSON-encoded by the handler.
165
+ """
166
+ # Build Bedrock Stability request
167
+ data: Dict[str, Any] = {
168
+ "output_format": "png", # Default to PNG
169
+ }
170
+
171
+ # Add prompt only if provided (some models don't require it)
172
+ if prompt is not None and prompt != "":
173
+ data["prompt"] = prompt
174
+
175
+ # Convert image to base64 if provided
176
+ if image is not None:
177
+ image_b64: str
178
+ if hasattr(image, "read") and callable(getattr(image, "read", None)):
179
+ # File-like object (e.g., BufferedReader from open())
180
+ image_bytes = image.read() # type: ignore
181
+ image_b64 = base64.b64encode(image_bytes).decode("utf-8") # type: ignore
182
+ elif isinstance(image, bytes):
183
+ # Raw bytes
184
+ image_b64 = base64.b64encode(image).decode("utf-8")
185
+ elif isinstance(image, str):
186
+ # Already a base64 string
187
+ image_b64 = image
188
+ else:
189
+ # Try to handle as bytes
190
+ image_b64 = base64.b64encode(bytes(image)).decode("utf-8") # type: ignore
191
+
192
+ # For style-transfer models, map image to init_image
193
+ model_lower = model.lower()
194
+ if "style-transfer" in model_lower:
195
+ data["init_image"] = image_b64
196
+ else:
197
+ data["image"] = image_b64
198
+
199
+ # Add optional params (already mapped in map_openai_params)
200
+ for key, value in image_edit_optional_request_params.items(): # type: ignore
201
+ # Skip internal params (prefixed with _)
202
+ if key.startswith("_") or value is None:
203
+ continue
204
+
205
+ # File-like optional params (mask, init_image, style_image, etc.)
206
+ if key in ["mask", "init_image", "style_image"]:
207
+ # Handle case where value might be in a list
208
+ file_value = value
209
+ if isinstance(value, list) and len(value) > 0:
210
+ file_value = value[0]
211
+
212
+ if hasattr(file_value, "read") and callable(
213
+ getattr(file_value, "read", None)
214
+ ):
215
+ file_bytes = file_value.read() # type: ignore
216
+ elif isinstance(file_value, bytes):
217
+ file_bytes = file_value
218
+ elif isinstance(file_value, str):
219
+ # Already a base64 string
220
+ data[key] = file_value
221
+ continue
222
+ else:
223
+ file_bytes = file_value # type: ignore
224
+
225
+ if isinstance(file_bytes, bytes):
226
+ file_b64 = base64.b64encode(file_bytes).decode("utf-8")
227
+ else:
228
+ file_b64 = str(file_bytes)
229
+ data[key] = file_b64
230
+ continue
231
+
232
+ # Numeric fields that need to be converted to int/float
233
+ numeric_int_fields = ["left", "right", "up", "down", "seed"]
234
+ numeric_float_fields = [
235
+ "strength",
236
+ "creativity",
237
+ "control_strength",
238
+ "grow_mask",
239
+ "fidelity",
240
+ "composition_fidelity",
241
+ "style_strength",
242
+ "change_strength",
243
+ ]
244
+
245
+ if key in numeric_int_fields:
246
+ # Convert to int (these are pixel values for outpaint)
247
+ try:
248
+ data[key] = int(value) # type: ignore
249
+ except (ValueError, TypeError):
250
+ data[key] = value # type: ignore
251
+ elif key in numeric_float_fields:
252
+ # Convert to float
253
+ try:
254
+ data[key] = float(value) # type: ignore
255
+ except (ValueError, TypeError):
256
+ data[key] = value # type: ignore
257
+
258
+ # Supported text fields
259
+ elif key in [
260
+ "negative_prompt",
261
+ "aspect_ratio",
262
+ "output_format",
263
+ "model",
264
+ "mode",
265
+ "style_preset",
266
+ "select_prompt",
267
+ "search_prompt",
268
+ ]:
269
+ data[key] = value # type: ignore
270
+
271
+ return data, {}
272
+
273
+ def transform_image_edit_response(
274
+ self,
275
+ model: str,
276
+ raw_response: httpx.Response,
277
+ logging_obj: LiteLLMLoggingObj,
278
+ api_key: Optional[str] = None,
279
+ json_mode: Optional[bool] = None,
280
+ ) -> ImageResponse:
281
+ """
282
+ Transform Bedrock Stability response to OpenAI-compatible ImageResponse.
283
+
284
+ Bedrock returns: {"images": ["base64..."], "finish_reasons": [null], "seeds": [123]}
285
+ OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp}
286
+ """
287
+ try:
288
+ response_data = raw_response.json()
289
+ except Exception as e:
290
+ raise self.get_error_class(
291
+ error_message=f"Error parsing Bedrock Stability response: {e}",
292
+ status_code=raw_response.status_code,
293
+ headers=raw_response.headers,
294
+ )
295
+
296
+ # Check for errors in response
297
+ if "errors" in response_data:
298
+ raise self.get_error_class(
299
+ error_message=f"Bedrock Stability error: {response_data['errors']}",
300
+ status_code=raw_response.status_code,
301
+ headers=raw_response.headers,
302
+ )
303
+
304
+ # Check finish_reasons
305
+ finish_reasons = response_data.get("finish_reasons", [])
306
+ if finish_reasons and finish_reasons[0]:
307
+ raise self.get_error_class(
308
+ error_message=f"Bedrock Stability error: {finish_reasons[0]}",
309
+ status_code=400,
310
+ headers=raw_response.headers,
311
+ )
312
+
313
+ model_response = ImageResponse()
314
+ if not model_response.data:
315
+ model_response.data = []
316
+
317
+ # Extract images from response
318
+ images = response_data.get("images", [])
319
+ if images:
320
+ for image_b64 in images:
321
+ if image_b64:
322
+ model_response.data.append(
323
+ ImageObject(
324
+ b64_json=image_b64,
325
+ url=None,
326
+ revised_prompt=None,
327
+ )
328
+ )
329
+
330
+ if not hasattr(model_response, "_hidden_params"):
331
+ model_response._hidden_params = {}
332
+ if "additional_headers" not in model_response._hidden_params:
333
+ model_response._hidden_params["additional_headers"] = {}
334
+
335
+ # Set cost based on model
336
+ model_info = get_model_info(model, custom_llm_provider="bedrock")
337
+ cost_per_image = model_info.get("output_cost_per_image", 0)
338
+ if cost_per_image is not None:
339
+ model_response._hidden_params["additional_headers"][
340
+ "llm_provider-x-litellm-response-cost"
341
+ ] = float(cost_per_image)
342
+
343
+ return model_response
344
+
345
+ def use_multipart_form_data(self) -> bool:
346
+ """
347
+ Bedrock Stability uses JSON format, not multipart/form-data.
348
+ """
349
+ return False
350
+
351
+ def get_complete_url(
352
+ self,
353
+ model: str,
354
+ api_base: Optional[str],
355
+ litellm_params: dict,
356
+ ) -> str:
357
+ """
358
+ Get the complete URL for the Bedrock Image Edit API.
359
+
360
+ For Bedrock, this is handled by the handler which constructs the endpoint URL
361
+ based on the model ID and AWS region. This method is required by the base class
362
+ but the actual URL construction happens in BedrockImageEdit.image_edit().
363
+
364
+ Returns a placeholder - the real endpoint is constructed in the handler.
365
+ """
366
+ # Bedrock URLs are constructed in the handler using boto3
367
+ # This is a placeholder for the abstract method requirement
368
+ return "bedrock://image-edit"
369
+
370
+ def validate_environment(
371
+ self,
372
+ headers: dict,
373
+ model: str,
374
+ api_key: Optional[str] = None,
375
+ ) -> dict:
376
+ """
377
+ Validate environment for Bedrock Stability image edit.
378
+
379
+ For Bedrock, AWS credentials are managed by the BaseAWSLLM class.
380
+ This method validates that headers are properly set up.
381
+
382
+ Args:
383
+ headers: The request headers to validate/update
384
+ model: The model name being used
385
+ api_key: Optional API key (not used for Bedrock, which uses AWS credentials)
386
+
387
+ Returns:
388
+ Updated headers dict
389
+ """
390
+ if headers is None:
391
+ headers = {}
392
+
393
+ # Bedrock uses AWS credentials, not API keys
394
+ # Headers are set up by the handler's get_request_headers() method
395
+ # This just ensures basic headers are present
396
+ if "Content-Type" not in headers:
397
+ headers["Content-Type"] = "application/json"
398
+
399
+ return headers
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py ADDED
@@ -0,0 +1,220 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import types
2
+ from typing import Any, Dict, List, Optional
3
+
4
+ from openai.types.image import Image
5
+
6
+ from litellm.types.llms.bedrock import (
7
+ AmazonNovaCanvasColorGuidedGenerationParams,
8
+ AmazonNovaCanvasColorGuidedRequest,
9
+ AmazonNovaCanvasImageGenerationConfig,
10
+ AmazonNovaCanvasInpaintingParams,
11
+ AmazonNovaCanvasInpaintingRequest,
12
+ AmazonNovaCanvasRequestBase,
13
+ AmazonNovaCanvasTextToImageParams,
14
+ AmazonNovaCanvasTextToImageRequest,
15
+ AmazonNovaCanvasTextToImageResponse,
16
+ )
17
+ from litellm.llms.bedrock.common_utils import get_cached_model_info
18
+ from litellm.types.utils import ImageResponse
19
+
20
+
21
+ class AmazonNovaCanvasConfig:
22
+ """
23
+ Reference: https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/model-catalog/serverless/amazon.nova-canvas-v1:0
24
+
25
+ """
26
+
27
+ @classmethod
28
+ def get_config(cls):
29
+ return {
30
+ k: v
31
+ for k, v in cls.__dict__.items()
32
+ if not k.startswith("__")
33
+ and not isinstance(
34
+ v,
35
+ (
36
+ types.FunctionType,
37
+ types.BuiltinFunctionType,
38
+ classmethod,
39
+ staticmethod,
40
+ ),
41
+ )
42
+ and v is not None
43
+ }
44
+
45
+ @classmethod
46
+ def get_supported_openai_params(cls, model: Optional[str] = None) -> List:
47
+ """ """
48
+ return ["n", "size", "quality"]
49
+
50
+ @classmethod
51
+ def _is_nova_model(cls, model: Optional[str] = None) -> bool:
52
+ """
53
+ Returns True if the model is a Nova Canvas model
54
+
55
+ Nova models follow this pattern:
56
+
57
+ """
58
+ if model and "amazon.nova-canvas" in model:
59
+ return True
60
+ return False
61
+
62
+ @classmethod
63
+ def transform_request_body(
64
+ cls, text: str, optional_params: dict
65
+ ) -> AmazonNovaCanvasRequestBase:
66
+ """
67
+ Transform the request body for Amazon Nova Canvas model
68
+ """
69
+ task_type = optional_params.pop("taskType", "TEXT_IMAGE")
70
+ image_generation_config = optional_params.pop("imageGenerationConfig", {})
71
+
72
+ # Extract model_id parameter to prevent "extraneous key" error from Bedrock API
73
+ # Following the same pattern as chat completions and embeddings
74
+ unencoded_model_id = optional_params.pop("model_id", None) # noqa: F841
75
+
76
+ image_generation_config = {**image_generation_config, **optional_params}
77
+ if task_type == "TEXT_IMAGE":
78
+ text_to_image_params: Dict[str, Any] = image_generation_config.pop(
79
+ "textToImageParams", {}
80
+ )
81
+ text_to_image_params = {"text": text, **text_to_image_params}
82
+ try:
83
+ text_to_image_params_typed = AmazonNovaCanvasTextToImageParams(
84
+ **text_to_image_params # type: ignore
85
+ )
86
+ except Exception as e:
87
+ raise ValueError(
88
+ f"Error transforming text to image params: {e}. Got params: {text_to_image_params}, Expected params: {AmazonNovaCanvasTextToImageParams.__annotations__}"
89
+ )
90
+
91
+ try:
92
+ image_generation_config_typed = AmazonNovaCanvasImageGenerationConfig(
93
+ **image_generation_config
94
+ )
95
+ except Exception as e:
96
+ raise ValueError(
97
+ f"Error transforming image generation config: {e}. Got params: {image_generation_config}, Expected params: {AmazonNovaCanvasImageGenerationConfig.__annotations__}"
98
+ )
99
+
100
+ return AmazonNovaCanvasTextToImageRequest(
101
+ textToImageParams=text_to_image_params_typed,
102
+ taskType=task_type,
103
+ imageGenerationConfig=image_generation_config_typed,
104
+ )
105
+ if task_type == "COLOR_GUIDED_GENERATION":
106
+ color_guided_generation_params: Dict[
107
+ str, Any
108
+ ] = image_generation_config.pop("colorGuidedGenerationParams", {})
109
+ color_guided_generation_params = {
110
+ "text": text,
111
+ **color_guided_generation_params,
112
+ }
113
+ try:
114
+ color_guided_generation_params_typed = AmazonNovaCanvasColorGuidedGenerationParams(
115
+ **color_guided_generation_params # type: ignore
116
+ )
117
+ except Exception as e:
118
+ raise ValueError(
119
+ f"Error transforming color guided generation params: {e}. Got params: {color_guided_generation_params}, Expected params: {AmazonNovaCanvasColorGuidedGenerationParams.__annotations__}"
120
+ )
121
+
122
+ try:
123
+ image_generation_config_typed = AmazonNovaCanvasImageGenerationConfig(
124
+ **image_generation_config
125
+ )
126
+ except Exception as e:
127
+ raise ValueError(
128
+ f"Error transforming image generation config: {e}. Got params: {image_generation_config}, Expected params: {AmazonNovaCanvasImageGenerationConfig.__annotations__}"
129
+ )
130
+
131
+ return AmazonNovaCanvasColorGuidedRequest(
132
+ taskType=task_type,
133
+ colorGuidedGenerationParams=color_guided_generation_params_typed,
134
+ imageGenerationConfig=image_generation_config_typed,
135
+ )
136
+ if task_type == "INPAINTING":
137
+ inpainting_params: Dict[str, Any] = image_generation_config.pop(
138
+ "inpaintingParams", {}
139
+ )
140
+ inpainting_params = {"text": text, **inpainting_params}
141
+ try:
142
+ inpainting_params_typed = AmazonNovaCanvasInpaintingParams(
143
+ **inpainting_params # type: ignore
144
+ )
145
+ except Exception as e:
146
+ raise ValueError(
147
+ f"Error transforming inpainting params: {e}. Got params: {inpainting_params}, Expected params: {AmazonNovaCanvasInpaintingParams.__annotations__}"
148
+ )
149
+
150
+ try:
151
+ image_generation_config_typed = AmazonNovaCanvasImageGenerationConfig(
152
+ **image_generation_config
153
+ )
154
+ except Exception as e:
155
+ raise ValueError(
156
+ f"Error transforming image generation config: {e}. Got params: {image_generation_config}, Expected params: {AmazonNovaCanvasImageGenerationConfig.__annotations__}"
157
+ )
158
+
159
+ return AmazonNovaCanvasInpaintingRequest(
160
+ taskType=task_type,
161
+ inpaintingParams=inpainting_params_typed,
162
+ imageGenerationConfig=image_generation_config_typed,
163
+ )
164
+ raise NotImplementedError(f"Task type {task_type} is not supported")
165
+
166
+ @classmethod
167
+ def map_openai_params(cls, non_default_params: dict, optional_params: dict) -> dict:
168
+ """
169
+ Map the OpenAI params to the Bedrock params
170
+ """
171
+ _size = non_default_params.get("size")
172
+ if _size is not None:
173
+ width, height = _size.split("x")
174
+ optional_params["width"], optional_params["height"] = int(width), int(
175
+ height
176
+ )
177
+ if non_default_params.get("n") is not None:
178
+ optional_params["numberOfImages"] = non_default_params.get("n")
179
+ if non_default_params.get("quality") is not None:
180
+ if non_default_params.get("quality") in ("hd", "premium"):
181
+ optional_params["quality"] = "premium"
182
+ if non_default_params.get("quality") == "standard":
183
+ optional_params["quality"] = "standard"
184
+ return optional_params
185
+
186
+ @classmethod
187
+ def transform_response_dict_to_openai_response(
188
+ cls, model_response: ImageResponse, response_dict: dict
189
+ ) -> ImageResponse:
190
+ """
191
+ Transform the response dict to the OpenAI response
192
+ """
193
+
194
+ nova_response = AmazonNovaCanvasTextToImageResponse(**response_dict)
195
+ openai_images: List[Image] = []
196
+ for _img in nova_response.get("images", []):
197
+ openai_images.append(Image(b64_json=_img))
198
+
199
+ model_response.data = openai_images
200
+ return model_response
201
+
202
+ @classmethod
203
+ def cost_calculator(
204
+ cls,
205
+ model: str,
206
+ image_response: ImageResponse,
207
+ size: Optional[str] = None,
208
+ optional_params: Optional[dict] = None,
209
+ ) -> float:
210
+ get_model_info = get_cached_model_info()
211
+ model_info = get_model_info(
212
+ model=model,
213
+ custom_llm_provider="bedrock",
214
+ )
215
+
216
+ output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
217
+ num_images: int = 0
218
+ if image_response.data:
219
+ num_images = len(image_response.data)
220
+ return output_cost_per_image * num_images
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import os
3
+ import types
4
+ from typing import List, Optional
5
+
6
+ from openai.types.image import Image
7
+
8
+ from litellm.llms.bedrock.common_utils import get_cached_model_info
9
+ from litellm.types.utils import ImageResponse
10
+
11
+
12
+ class AmazonStabilityConfig:
13
+ """
14
+ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=stability.stable-diffusion-xl-v0
15
+
16
+ Supported Params for the Amazon / Stable Diffusion models:
17
+
18
+ - `cfg_scale` (integer): Default `7`. Between [ 0 .. 35 ]. How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt)
19
+
20
+ - `seed` (float): Default: `0`. Between [ 0 .. 4294967295 ]. Random noise seed (omit this option or use 0 for a random seed)
21
+
22
+ - `steps` (array of strings): Default `30`. Between [ 10 .. 50 ]. Number of diffusion steps to run.
23
+
24
+ - `width` (integer): Default: `512`. multiple of 64 >= 128. Width of the image to generate, in pixels, in an increment divible by 64.
25
+ Engine-specific dimension validation:
26
+
27
+ - SDXL Beta: must be between 128x128 and 512x896 (or 896x512); only one dimension can be greater than 512.
28
+ - SDXL v0.9: must be one of 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640, 640x1536, 768x1344, 832x1216, or 896x1152
29
+ - SDXL v1.0: same as SDXL v0.9
30
+ - SD v1.6: must be between 320x320 and 1536x1536
31
+
32
+ - `height` (integer): Default: `512`. multiple of 64 >= 128. Height of the image to generate, in pixels, in an increment divible by 64.
33
+ Engine-specific dimension validation:
34
+
35
+ - SDXL Beta: must be between 128x128 and 512x896 (or 896x512); only one dimension can be greater than 512.
36
+ - SDXL v0.9: must be one of 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640, 640x1536, 768x1344, 832x1216, or 896x1152
37
+ - SDXL v1.0: same as SDXL v0.9
38
+ - SD v1.6: must be between 320x320 and 1536x1536
39
+ """
40
+
41
+ cfg_scale: Optional[int] = None
42
+ seed: Optional[float] = None
43
+ steps: Optional[List[str]] = None
44
+ width: Optional[int] = None
45
+ height: Optional[int] = None
46
+
47
+ def __init__(
48
+ self,
49
+ cfg_scale: Optional[int] = None,
50
+ seed: Optional[float] = None,
51
+ steps: Optional[List[str]] = None,
52
+ width: Optional[int] = None,
53
+ height: Optional[int] = None,
54
+ ) -> None:
55
+ locals_ = locals().copy()
56
+ for key, value in locals_.items():
57
+ if key != "self" and value is not None:
58
+ setattr(self.__class__, key, value)
59
+
60
+ @classmethod
61
+ def get_config(cls):
62
+ return {
63
+ k: v
64
+ for k, v in cls.__dict__.items()
65
+ if not k.startswith("__")
66
+ and not isinstance(
67
+ v,
68
+ (
69
+ types.FunctionType,
70
+ types.BuiltinFunctionType,
71
+ classmethod,
72
+ staticmethod,
73
+ ),
74
+ )
75
+ and v is not None
76
+ }
77
+
78
+ @classmethod
79
+ def get_supported_openai_params(cls, model: Optional[str] = None) -> List:
80
+ return ["size"]
81
+
82
+ @classmethod
83
+ def map_openai_params(
84
+ cls,
85
+ non_default_params: dict,
86
+ optional_params: dict,
87
+ ):
88
+ _size = non_default_params.get("size")
89
+ if _size is not None:
90
+ width, height = _size.split("x")
91
+ optional_params["width"] = int(width)
92
+ optional_params["height"] = int(height)
93
+
94
+ return optional_params
95
+
96
+ @classmethod
97
+ def transform_request_body(
98
+ cls,
99
+ text: str,
100
+ optional_params: dict,
101
+ ) -> dict:
102
+ inference_params = copy.deepcopy(optional_params)
103
+ inference_params.pop(
104
+ "user", None
105
+ ) # make sure user is not passed in for bedrock call
106
+
107
+ prompt = text.replace(os.linesep, " ")
108
+ ## LOAD CONFIG
109
+ config = cls.get_config()
110
+ for k, v in config.items():
111
+ if (
112
+ k not in inference_params
113
+ ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
114
+ inference_params[k] = v
115
+
116
+ return {
117
+ "text_prompts": [{"text": prompt, "weight": 1}],
118
+ **inference_params,
119
+ }
120
+
121
+ @classmethod
122
+ def transform_response_dict_to_openai_response(
123
+ cls, model_response: ImageResponse, response_dict: dict
124
+ ) -> ImageResponse:
125
+ image_list: List[Image] = []
126
+ for artifact in response_dict["artifacts"]:
127
+ _image = Image(b64_json=artifact["base64"])
128
+ image_list.append(_image)
129
+
130
+ model_response.data = image_list
131
+
132
+ return model_response
133
+
134
+ @classmethod
135
+ def cost_calculator(
136
+ cls,
137
+ model: str,
138
+ image_response: ImageResponse,
139
+ size: Optional[str] = None,
140
+ optional_params: Optional[dict] = None,
141
+ ) -> float:
142
+ optional_params = optional_params or {}
143
+
144
+ # see model_prices_and_context_window.json for details on how steps is used
145
+ # Reference pricing by steps for stability 1: https://aws.amazon.com/bedrock/pricing/
146
+ _steps = optional_params.get("steps", 50)
147
+ steps = "max-steps" if _steps > 50 else "50-steps"
148
+
149
+ # size is stored in model_prices_and_context_window.json as 1024-x-1024
150
+ # current size has 1024x1024
151
+ size = size or "1024-x-1024"
152
+ model = f"{size}/{steps}/{model}"
153
+
154
+ get_model_info = get_cached_model_info()
155
+ model_info = get_model_info(
156
+ model=model,
157
+ custom_llm_provider="bedrock",
158
+ )
159
+
160
+ output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
161
+ num_images: int = 0
162
+ if image_response.data:
163
+ num_images = len(image_response.data)
164
+ return output_cost_per_image * num_images
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import types
2
+ from typing import List, Optional
3
+
4
+ from openai.types.image import Image
5
+
6
+ from litellm.llms.bedrock.common_utils import BedrockError
7
+ from litellm.types.llms.bedrock import (
8
+ AmazonStability3TextToImageRequest,
9
+ AmazonStability3TextToImageResponse,
10
+ )
11
+ from litellm.llms.bedrock.common_utils import get_cached_model_info
12
+ from litellm.types.utils import ImageResponse
13
+
14
+
15
+ class AmazonStability3Config:
16
+ """
17
+ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=stability.stable-diffusion-xl-v0
18
+
19
+ Stability API Ref: https://platform.stability.ai/docs/api-reference#tag/Generate/paths/~1v2beta~1stable-image~1generate~1sd3/post
20
+ """
21
+
22
+ @classmethod
23
+ def get_config(cls):
24
+ return {
25
+ k: v
26
+ for k, v in cls.__dict__.items()
27
+ if not k.startswith("__")
28
+ and not isinstance(
29
+ v,
30
+ (
31
+ types.FunctionType,
32
+ types.BuiltinFunctionType,
33
+ classmethod,
34
+ staticmethod,
35
+ ),
36
+ )
37
+ and v is not None
38
+ }
39
+
40
+ @classmethod
41
+ def get_supported_openai_params(cls, model: Optional[str] = None) -> List:
42
+ """
43
+ No additional OpenAI params are mapped for stability 3
44
+ """
45
+ return []
46
+
47
+ @classmethod
48
+ def _is_stability_3_model(cls, model: Optional[str] = None) -> bool:
49
+ """
50
+ Returns True if the model is a Stability 3 model
51
+
52
+ Stability 3 models follow this pattern:
53
+ sd3-large
54
+ sd3-large-turbo
55
+ sd3-medium
56
+ sd3.5-large
57
+ sd3.5-large-turbo
58
+
59
+ Stability ultra models
60
+ stable-image-ultra-v1
61
+ """
62
+ if model:
63
+ if "sd3" in model or "sd3.5" in model:
64
+ return True
65
+ if "stable-image" in model:
66
+ return True
67
+ return False
68
+
69
+ @classmethod
70
+ def transform_request_body(
71
+ cls, text: str, optional_params: dict
72
+ ) -> AmazonStability3TextToImageRequest:
73
+ """
74
+ Transform the request body for the Stability 3 models
75
+ """
76
+ data = AmazonStability3TextToImageRequest(prompt=text, **optional_params)
77
+ return data
78
+
79
+ @classmethod
80
+ def map_openai_params(cls, non_default_params: dict, optional_params: dict) -> dict:
81
+ """
82
+ Map the OpenAI params to the Bedrock params
83
+
84
+ No OpenAI params are mapped for Stability 3, so directly return the optional_params
85
+ """
86
+ return optional_params
87
+
88
+ @classmethod
89
+ def transform_response_dict_to_openai_response(
90
+ cls, model_response: ImageResponse, response_dict: dict
91
+ ) -> ImageResponse:
92
+ """
93
+ Transform the response dict to the OpenAI response
94
+ """
95
+
96
+ stability_3_response = AmazonStability3TextToImageResponse(**response_dict)
97
+
98
+ finish_reasons = stability_3_response.get("finish_reasons", [])
99
+ finish_reasons = [reason for reason in finish_reasons if reason]
100
+ if len(finish_reasons) > 0:
101
+ raise BedrockError(status_code=400, message="; ".join(finish_reasons))
102
+
103
+ openai_images: List[Image] = []
104
+ for _img in stability_3_response.get("images", []):
105
+ openai_images.append(Image(b64_json=_img))
106
+
107
+ model_response.data = openai_images
108
+ return model_response
109
+
110
+ @classmethod
111
+ def cost_calculator(
112
+ cls,
113
+ model: str,
114
+ image_response: ImageResponse,
115
+ size: Optional[str] = None,
116
+ optional_params: Optional[dict] = None,
117
+ ) -> float:
118
+ get_model_info = get_cached_model_info()
119
+ model_info = get_model_info(
120
+ model=model,
121
+ custom_llm_provider="bedrock",
122
+ )
123
+
124
+ output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
125
+ num_images: int = 0
126
+ if image_response.data:
127
+ num_images = len(image_response.data)
128
+ return output_cost_per_image * num_images
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation logic for Amazon Titan Image Generation.
3
+ """
4
+
5
+ import types
6
+ from typing import List, Optional
7
+
8
+ from openai.types.image import Image
9
+
10
+ from litellm.utils import get_model_info
11
+ from litellm.types.llms.bedrock import (
12
+ AmazonNovaCanvasImageGenerationConfig,
13
+ AmazonTitanImageGenerationRequestBody,
14
+ AmazonTitanTextToImageParams,
15
+ )
16
+ from litellm.types.utils import ImageResponse
17
+
18
+
19
+ class AmazonTitanImageGenerationConfig:
20
+ """
21
+ Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=stability.stable-diffusion-xl-v0
22
+ """
23
+
24
+ cfg_scale: Optional[int] = None
25
+ seed: Optional[float] = None
26
+ steps: Optional[List[str]] = None
27
+ width: Optional[int] = None
28
+ height: Optional[int] = None
29
+
30
+ def __init__(
31
+ self,
32
+ cfg_scale: Optional[int] = None,
33
+ seed: Optional[float] = None,
34
+ steps: Optional[List[str]] = None,
35
+ width: Optional[int] = None,
36
+ height: Optional[int] = None,
37
+ ) -> None:
38
+ locals_ = locals().copy()
39
+ for key, value in locals_.items():
40
+ if key != "self" and value is not None:
41
+ setattr(self.__class__, key, value)
42
+
43
+ @classmethod
44
+ def get_config(cls):
45
+ return {
46
+ k: v
47
+ for k, v in cls.__dict__.items()
48
+ if not k.startswith("__")
49
+ and not isinstance(
50
+ v,
51
+ (
52
+ types.FunctionType,
53
+ types.BuiltinFunctionType,
54
+ classmethod,
55
+ staticmethod,
56
+ ),
57
+ )
58
+ and v is not None
59
+ }
60
+
61
+ @classmethod
62
+ def _is_titan_model(cls, model: Optional[str] = None) -> bool:
63
+ """
64
+ Returns True if the model is a Titan model
65
+
66
+ Titan models follow this pattern:
67
+
68
+ """
69
+ if model and "amazon.titan" in model:
70
+ return True
71
+ return False
72
+
73
+ @classmethod
74
+ def get_supported_openai_params(cls, model: Optional[str] = None) -> List:
75
+ return ["size", "n", "quality"]
76
+
77
+ @classmethod
78
+ def map_openai_params(
79
+ cls,
80
+ non_default_params: dict,
81
+ optional_params: dict,
82
+ ):
83
+ from typing import Any, Dict
84
+
85
+ image_generation_config: Dict[str, Any] = {}
86
+ for k, v in non_default_params.items():
87
+ if k == "size" and v is not None:
88
+ width, height = v.split("x")
89
+ image_generation_config["width"] = int(width)
90
+ image_generation_config["height"] = int(height)
91
+ elif k == "n" and v is not None:
92
+ image_generation_config["numberOfImages"] = v
93
+ elif (
94
+ k == "quality" and v is not None
95
+ ): # 'auto', 'hd', 'standard', 'high', 'medium', 'low'
96
+ if v in ("hd", "premium", "high"):
97
+ image_generation_config["quality"] = "premium"
98
+ elif v in ("standard", "medium", "low"):
99
+ image_generation_config["quality"] = "standard"
100
+
101
+ if image_generation_config:
102
+ optional_params["imageGenerationConfig"] = image_generation_config
103
+ return optional_params
104
+
105
+ @classmethod
106
+ def transform_request_body(
107
+ cls,
108
+ text: str,
109
+ optional_params: dict,
110
+ ) -> AmazonTitanImageGenerationRequestBody:
111
+ from typing import Any, Dict
112
+
113
+ image_generation_config = optional_params.pop("imageGenerationConfig", {})
114
+ negative_text = optional_params.pop("negativeText", None)
115
+ text_to_image_params: Dict[str, Any] = {"text": text}
116
+ if negative_text:
117
+ text_to_image_params["negativeText"] = negative_text
118
+ task_type = optional_params.pop("taskType", "TEXT_IMAGE")
119
+ user_specified_image_generation_config = optional_params.pop(
120
+ "imageGenerationConfig", {}
121
+ )
122
+ image_generation_config = {
123
+ **image_generation_config,
124
+ **user_specified_image_generation_config,
125
+ }
126
+ return AmazonTitanImageGenerationRequestBody(
127
+ taskType=task_type,
128
+ textToImageParams=AmazonTitanTextToImageParams(**text_to_image_params), # type: ignore
129
+ imageGenerationConfig=AmazonNovaCanvasImageGenerationConfig(
130
+ **image_generation_config
131
+ ),
132
+ )
133
+
134
+ @classmethod
135
+ def transform_response_dict_to_openai_response(
136
+ cls, model_response: ImageResponse, response_dict: dict
137
+ ) -> ImageResponse:
138
+ image_list: List[Image] = []
139
+ for image in response_dict["images"]:
140
+ _image = Image(b64_json=image)
141
+ image_list.append(_image)
142
+
143
+ model_response.data = image_list
144
+
145
+ return model_response
146
+
147
+ @classmethod
148
+ def cost_calculator(
149
+ cls,
150
+ model: str,
151
+ image_response: ImageResponse,
152
+ size: Optional[str] = None,
153
+ optional_params: Optional[dict] = None,
154
+ ) -> float:
155
+ model_info = get_model_info(model=model)
156
+ output_cost_per_image = model_info.get("output_cost_per_image") or 0.0
157
+ if not image_response.data:
158
+ return 0.0
159
+ num_images = len(image_response.data)
160
+ return output_cost_per_image * num_images
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/image_generation/cost_calculator.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+
3
+ from litellm.llms.bedrock.image_generation.image_handler import BedrockImageGeneration
4
+ from litellm.types.utils import ImageResponse
5
+
6
+
7
+ def cost_calculator(
8
+ model: str,
9
+ image_response: ImageResponse,
10
+ size: Optional[str] = None,
11
+ optional_params: Optional[dict] = None,
12
+ ) -> float:
13
+ """
14
+ Bedrock image generation cost calculator
15
+
16
+ Handles both Stability 1 and Stability 3 models
17
+ """
18
+ config_class = BedrockImageGeneration.get_config_class(model=model)
19
+ return config_class.cost_calculator(
20
+ model=model,
21
+ image_response=image_response,
22
+ size=size,
23
+ optional_params=optional_params,
24
+ )