diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/baseten/chat.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/baseten/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..1e49b3460886e4f2bedc37cd962490bdeb261d4d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/baseten/chat.py @@ -0,0 +1,121 @@ +from typing import Optional +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + +class BasetenConfig(OpenAIGPTConfig): + """ + Reference: https://inference.baseten.co/v1 + + Below are the parameters: + """ + + max_tokens: Optional[int] = None + response_format: Optional[dict] = None + seed: Optional[int] = None + stream: Optional[bool] = None + top_p: Optional[int] = None + tool_choice: Optional[str] = None + tools: Optional[list] = None + user: Optional[str] = None + presence_penalty: Optional[int] = None + frequency_penalty: Optional[int] = None + stream_options: Optional[dict] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + response_format: Optional[dict] = None, + seed: Optional[int] = None, + stop: Optional[list] = None, + stream: Optional[bool] = None, + temperature: Optional[float] = None, + top_p: Optional[int] = None, + tool_choice: Optional[str] = None, + tools: Optional[list] = None, + user: Optional[str] = None, + presence_penalty: Optional[int] = None, + frequency_penalty: Optional[int] = None, + stream_options: Optional[dict] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return super().get_config() + + def get_supported_openai_params(self, model: str) -> list: + """ + Get the supported OpenAI params for the given model + """ + return [ + "max_tokens", + "max_completion_tokens", + "response_format", + "seed", + "stop", + "stream", + "temperature", + "top_p", + "tool_choice", + "tools", + "user", + "presence_penalty", + "frequency_penalty", + "stream_options", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_openai_params = self.get_supported_openai_params(model=model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_openai_params: + optional_params[param] = value + return optional_params + + def _get_openai_compatible_provider_info( + self, api_base: str, api_key: str + ) -> tuple: + """ + Get the OpenAI compatible provider info for Baseten + """ + # Default to Model API + default_api_base = "https://inference.baseten.co/v1" + default_api_key = api_key or "BASETEN_API_KEY" + + return default_api_base, default_api_key + + @staticmethod + def is_dedicated_deployment(model: str) -> bool: + """ + Check if the model is a dedicated deployment (8-digit alphanumeric code) + """ + # Remove 'baseten/' prefix if present + model_id = model.replace("baseten/", "") + + # Check if it's an 8-digit alphanumeric code + import re + + return bool(re.match(r"^[a-zA-Z0-9]{8}$", model_id)) + + @staticmethod + def get_api_base_for_model(model: str) -> str: + """ + Get the appropriate API base URL for the given model + """ + if BasetenConfig.is_dedicated_deployment(model): + # Extract the model ID (remove 'baseten/' prefix if present) + model_id = model.replace("baseten/", "") + return f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" + else: + # Use Model API + return "https://inference.baseten.co/v1" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/base_aws_llm.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/base_aws_llm.py new file mode 100644 index 0000000000000000000000000000000000000000..b159d62367d7ddbab65fd595c5f7726c83cfd621 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/base_aws_llm.py @@ -0,0 +1,1406 @@ +import hashlib +import json +import os +import urllib.parse +from datetime import datetime +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Optional, + Tuple, + Union, + cast, + get_args, +) + +import httpx +from pydantic import BaseModel + +from litellm._logging import verbose_logger +from litellm.caching.caching import DualCache +from litellm.constants import ( + BEDROCK_EMBEDDING_PROVIDERS_LITERAL, + BEDROCK_INVOKE_PROVIDERS_LITERAL, + BEDROCK_MAX_POLICY_SIZE, +) +from litellm.litellm_core_utils.dd_tracing import tracer +from litellm.secret_managers.main import get_secret, get_secret_str + +if TYPE_CHECKING: + from botocore.awsrequest import AWSPreparedRequest + from botocore.credentials import Credentials +else: + Credentials = Any + AWSPreparedRequest = Any + + +class Boto3CredentialsInfo(BaseModel): + credentials: Credentials + aws_region_name: str + aws_bedrock_runtime_endpoint: Optional[str] + + +class AwsAuthError(Exception): + def __init__(self, status_code, message): + self.status_code = status_code + self.message = message + self.request = httpx.Request( + method="POST", url="https://us-west-2.console.aws.amazon.com/bedrock" + ) + self.response = httpx.Response(status_code=status_code, request=self.request) + super().__init__( + self.message + ) # Call the base class constructor with the parameters it needs + + +class BaseAWSLLM: + def __init__(self) -> None: + self.iam_cache = DualCache() + super().__init__() + self.aws_authentication_params = [ + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_region_name", + "aws_session_name", + "aws_profile_name", + "aws_role_name", + "aws_web_identity_token", + "aws_sts_endpoint", + "aws_bedrock_runtime_endpoint", + "aws_external_id", + ] + + def _get_ssl_verify(self, ssl_verify: Optional[Union[bool, str]] = None): + """ + Get SSL verification setting for boto3 clients. + + This ensures that custom CA certificates are properly used for all AWS API calls, + including STS and Bedrock services. + + Returns: + Union[bool, str]: SSL verification setting - False to disable, True to enable, + or a string path to a CA bundle file + """ + from litellm.llms.custom_httpx.http_handler import get_ssl_verify + + return get_ssl_verify(ssl_verify=ssl_verify) + + def get_cache_key(self, credential_args: Dict[str, Optional[str]]) -> str: + """ + Generate a unique cache key based on the credential arguments. + """ + # Convert credential arguments to a JSON string and hash it to create a unique key + credential_str = json.dumps(credential_args, sort_keys=True) + return hashlib.sha256(credential_str.encode()).hexdigest() + + @tracer.wrap() + def get_credentials( + self, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + aws_region_name: Optional[str] = None, + aws_session_name: Optional[str] = None, + aws_profile_name: Optional[str] = None, + aws_role_name: Optional[str] = None, + aws_web_identity_token: Optional[str] = None, + aws_sts_endpoint: Optional[str] = None, + aws_external_id: Optional[str] = None, + ssl_verify: Optional[Union[bool, str]] = None, + ): + """ + Return a boto3.Credentials object + """ + ## CHECK IS 'os.environ/' passed in + params_to_check: List[Optional[str]] = [ + aws_access_key_id, + aws_secret_access_key, + aws_session_token, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + aws_sts_endpoint, + aws_external_id, + ] + + # Iterate over parameters and update if needed + for i, param in enumerate(params_to_check): + if param and param.startswith("os.environ/"): + _v = get_secret(param) + if _v is not None and isinstance(_v, str): + params_to_check[i] = _v + elif param is None: # check if uppercase value in env + key = self.aws_authentication_params[i] + if key.upper() in os.environ: + params_to_check[i] = os.getenv(key.upper()) + + # Assign updated values back to parameters + ( + aws_access_key_id, + aws_secret_access_key, + aws_session_token, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + aws_sts_endpoint, + aws_external_id, + ) = params_to_check + + verbose_logger.debug( + "in get credentials\n" + "aws_access_key_id=%s\n" + "aws_secret_access_key=%s\n" + "aws_session_token=%s\n" + "aws_region_name=%s\n" + "aws_session_name=%s\n" + "aws_profile_name=%s\n" + "aws_role_name=%s\n" + "aws_web_identity_token=%s\n" + "aws_sts_endpoint=%s\n" + "aws_external_id=%s", + aws_access_key_id, + aws_secret_access_key, + aws_session_token, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + aws_sts_endpoint, + aws_external_id, + ) + + # create cache key for non-expiring auth flows + args = { + k: v + for k, v in locals().items() + if k.startswith("aws_") or k == "ssl_verify" + } + + cache_key = self.get_cache_key(args) + _cached_credentials = self.iam_cache.get_cache(cache_key) + if _cached_credentials: + return _cached_credentials + + ######################################################### + # Handle diff boto3 auth flows + # for each helper + # Return: + # Credentials - boto3.Credentials + # cache ttl - Optional[int]. If None, the credentials are not cached. Some auth flows have no expiry time. + ######################################################### + if ( + aws_web_identity_token is not None + and aws_role_name is not None + and aws_session_name is not None + ): + credentials, _cache_ttl = self._auth_with_web_identity_token( + aws_web_identity_token=aws_web_identity_token, + aws_role_name=aws_role_name, + aws_session_name=aws_session_name, + aws_region_name=aws_region_name, + aws_sts_endpoint=aws_sts_endpoint, + aws_external_id=aws_external_id, + ) + elif aws_role_name is not None: + # Check if we're already running as the target role and can skip assumption + # This handles IRSA (EKS), ECS task roles, and EC2 instance profiles + if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify): + verbose_logger.debug( + "Already running as target role %s, using ambient credentials", + aws_role_name, + ) + credentials, _cache_ttl = self._auth_with_env_vars() + else: + verbose_logger.debug( + "Using role assumption: calling _auth_with_aws_role" + ) + # If aws_session_name is not provided, generate a default one + if aws_session_name is None: + aws_session_name = ( + f"litellm-session-{int(datetime.now().timestamp())}" + ) + credentials, _cache_ttl = self._auth_with_aws_role( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_role_name=aws_role_name, + aws_session_name=aws_session_name, + aws_region_name=aws_region_name, + aws_sts_endpoint=aws_sts_endpoint, + aws_external_id=aws_external_id, + ssl_verify=ssl_verify, + ) + + elif aws_profile_name is not None: ### CHECK SESSION ### + credentials, _cache_ttl = self._auth_with_aws_profile(aws_profile_name) + elif ( + aws_access_key_id is not None + and aws_secret_access_key is not None + and aws_session_token is not None + ): + credentials, _cache_ttl = self._auth_with_aws_session_token( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + ) + elif ( + aws_access_key_id is not None + and aws_secret_access_key is not None + and aws_region_name is not None + ): + credentials, _cache_ttl = self._auth_with_access_key_and_secret_key( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_region_name=aws_region_name, + ) + else: + credentials, _cache_ttl = self._auth_with_env_vars() + + self.iam_cache.set_cache(cache_key, credentials, ttl=_cache_ttl) + return credentials + + def _get_aws_region_from_model_arn(self, model: Optional[str]) -> Optional[str]: + try: + # First check if the string contains the expected prefix + if not isinstance(model, str) or "arn:aws:bedrock" not in model: + return None + + # Split the ARN and check if we have enough parts + parts = model.split(":") + if len(parts) < 4: + return None + + # Get the region from the correct position + region = parts[3] + if not region: # Check if region is empty + return None + + return region + except Exception: + # Catch any unexpected errors and return None + return None + + @staticmethod + def _get_provider_from_model_path( + model_path: str, + ) -> Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL]: + """ + Helper function to get the provider from a model path with format: provider/model-name + + Args: + model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name') + + Returns: + Optional[str]: The provider name, or None if no valid provider found + """ + parts = model_path.split("/") + if len(parts) >= 1: + provider = parts[0] + if provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) + return None + + @staticmethod + def get_bedrock_invoke_provider( + model: str, + ) -> Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL]: + """ + Helper function to get the bedrock provider from the model + + handles 3 scenarions: + 1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` + 2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` + 3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama` + 4. model=us.amazon.nova-pro-v1:0 -> Returns `nova` + """ + if model.startswith("invoke/"): + model = model.replace("invoke/", "", 1) + + # Special case: Check for "nova" in model name first (before "amazon") + # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) + if "nova" in model.lower(): + if "nova" in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") + + _split_model = model.split(".")[0] + if _split_model in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) + + # If not a known provider, check for pattern with two slashes + provider = BaseAWSLLM._get_provider_from_model_path(model) + if provider is not None: + return provider + + for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + if provider in model: + return provider + return None + + @staticmethod + def get_bedrock_model_id( + optional_params: dict, + provider: Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL], + model: str, + ) -> str: + model_id = optional_params.pop("model_id", None) + if model_id is not None: + model_id = BaseAWSLLM.encode_model_id(model_id=model_id) + else: + model_id = model + + model_id = model_id.replace("invoke/", "", 1) + if provider == "llama" and "llama/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="llama" + ) + elif provider == "deepseek_r1" and "deepseek_r1/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="deepseek_r1" + ) + elif provider == "openai" and "openai/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="openai" + ) + elif provider == "qwen2" and "qwen2/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="qwen2" + ) + elif provider == "qwen3" and "qwen3/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="qwen3" + ) + elif provider == "stability" and "stability/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="stability" + ) + elif provider == "moonshot" and "moonshot/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="moonshot" + ) + elif "nova-2/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="nova-2" + ) + elif "nova/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="nova" + ) + return model_id + + @staticmethod + def _get_model_id_from_model_with_spec( + model: str, + spec: str, + ) -> str: + """ + Remove `llama` from modelID since `llama` is simply a spec to follow for custom bedrock models + """ + model_id = model.replace(spec + "/", "") + return BaseAWSLLM.encode_model_id(model_id=model_id) + + @staticmethod + def encode_model_id(model_id: str) -> str: + """ + Double encode the model ID to ensure it matches the expected double-encoded format. + Args: + model_id (str): The model ID to encode. + Returns: + str: The double-encoded model ID. + """ + return urllib.parse.quote(model_id, safe="") + + @staticmethod + def get_bedrock_embedding_provider( + model: str, + ) -> Optional[BEDROCK_EMBEDDING_PROVIDERS_LITERAL]: + """ + Helper function to get the bedrock embedding provider from the model + + Handles scenarios like: + 1. model=cohere.embed-english-v3:0 -> Returns `cohere` + 2. model=amazon.titan-embed-text-v1 -> Returns `amazon` + 3. model=amazon.nova-2-multimodal-embeddings-v1:0 -> Returns `nova` + 4. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs` + 5. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs` + """ + # Special case: Check for "nova" in model name first (before "amazon") + # This handles amazon.nova-* models + if "nova" in model.lower(): + if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): + return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova") + + # Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0 + if "." in model: + parts = model.split(".") + # Check if the second part (after potential region) is a known provider + if len(parts) >= 2: + potential_provider = parts[ + 1 + ] # e.g., "twelvelabs" from "us.twelvelabs.marengo-embed-2-7-v1:0" + if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): + return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider) + + # Check if the first part is a known provider (standard format) + potential_provider = parts[ + 0 + ] # e.g., "cohere" from "cohere.embed-english-v3:0" + if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): + return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider) + + # Fallback: check if any provider name appears in the model string + for provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): + if provider in model: + return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, provider) + + return None + + def _get_aws_region_name( + self, + optional_params: dict, + model: Optional[str] = None, + model_id: Optional[str] = None, + ) -> str: + """ + Get the AWS region name from the environment variables. + + Parameters: + optional_params (dict): Optional parameters for the model call + model (str): The model name + model_id (str): The model ID. This is the ARN of the model, if passed in as a separate param. + + Returns: + str: The AWS region name + """ + aws_region_name = optional_params.get("aws_region_name", None) + ### SET REGION NAME ### + if aws_region_name is None: + # check model arn # + if model_id is not None: + aws_region_name = self._get_aws_region_from_model_arn(model_id) + else: + aws_region_name = self._get_aws_region_from_model_arn(model) + # check env # + litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) + + if ( + aws_region_name is None + and litellm_aws_region_name is not None + and isinstance(litellm_aws_region_name, str) + ): + aws_region_name = litellm_aws_region_name + + standard_aws_region_name = get_secret("AWS_REGION", None) + if ( + aws_region_name is None + and standard_aws_region_name is not None + and isinstance(standard_aws_region_name, str) + ): + aws_region_name = standard_aws_region_name + if aws_region_name is None: + try: + import boto3 + + with tracer.trace("boto3.Session()"): + session = boto3.Session() + configured_region = session.region_name + if configured_region: + aws_region_name = configured_region + else: + aws_region_name = "us-west-2" + except Exception: + aws_region_name = "us-west-2" + + return aws_region_name + + def get_aws_region_name_for_non_llm_api_calls( + self, + aws_region_name: Optional[str] = None, + ): + """ + Get the AWS region name for non-llm api calls. + + LLM API calls check the model arn and end up using that as the region name. + + For non-llm api calls eg. Guardrails, Vector Stores we just need to check the dynamic param or env vars. + """ + if aws_region_name is None: + # check env # + litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) + + if litellm_aws_region_name is not None and isinstance( + litellm_aws_region_name, str + ): + aws_region_name = litellm_aws_region_name + + standard_aws_region_name = get_secret("AWS_REGION", None) + if standard_aws_region_name is not None and isinstance( + standard_aws_region_name, str + ): + aws_region_name = standard_aws_region_name + + if aws_region_name is None: + aws_region_name = "us-west-2" + return aws_region_name + + @staticmethod + def _parse_arn_account_and_role_name( + arn: str, + ) -> Optional[Tuple[str, str, str]]: + """ + Parse an ARN and return (partition, account_id, role_name). + + Handles: + - arn:aws:iam::123456789012:role/MyRole + - arn:aws:iam::123456789012:role/path/to/MyRole + - arn:aws:sts::123456789012:assumed-role/MyRole/session-name + + Returns None if the ARN cannot be parsed. + """ + # ARN format: arn:PARTITION:SERVICE:REGION:ACCOUNT:RESOURCE + parts = arn.split(":") + if len(parts) < 6 or parts[0] != "arn": + return None + + partition = parts[1] # e.g. "aws", "aws-cn", "aws-us-gov" + account_id = parts[4] + resource = ":".join(parts[5:]) # rejoin in case resource contains colons + + if resource.startswith("role/"): + # arn:aws:iam::ACCOUNT:role/[path/]ROLE_NAME + role_name = resource.split("/")[-1] + elif resource.startswith("assumed-role/"): + # arn:aws:sts::ACCOUNT:assumed-role/ROLE_NAME/SESSION + role_parts = resource.split("/") + if len(role_parts) >= 2: + role_name = role_parts[1] + else: + return None + else: + return None + + return partition, account_id, role_name + + def _is_already_running_as_role( + self, + aws_role_name: str, + ssl_verify: Optional[Union[bool, str]] = None, + ) -> bool: + """ + Check if the current environment is already running as the target IAM role. + + This handles multiple AWS environments: + - IRSA (EKS): AWS_ROLE_ARN + AWS_WEB_IDENTITY_TOKEN_FILE are set + - ECS task roles: Uses sts:GetCallerIdentity to check current role ARN + - EC2 instance profiles: Uses sts:GetCallerIdentity to check current role ARN + + Compares partition, account ID, and role name to avoid cross-account + false matches. + + Returns True if the current identity matches the target role, meaning + we can skip sts:AssumeRole and use ambient credentials directly. + """ + target_parsed = self._parse_arn_account_and_role_name(aws_role_name) + if target_parsed is None: + return False + + target_partition, target_account, target_role = target_parsed + + # Fast path: IRSA environment check (no API call needed) + current_role_arn = os.getenv("AWS_ROLE_ARN") + web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE") + if current_role_arn and web_identity_token_file: + return current_role_arn == aws_role_name + + # For ECS/EC2: call sts:GetCallerIdentity to check if already running as the role + try: + import boto3 + + with tracer.trace("boto3.client(sts).get_caller_identity"): + sts_client = boto3.client( + "sts", verify=self._get_ssl_verify(ssl_verify) + ) + identity = sts_client.get_caller_identity() + caller_arn = identity.get("Arn", "") + + caller_parsed = self._parse_arn_account_and_role_name(caller_arn) + if caller_parsed is not None: + caller_partition, caller_account, caller_role = caller_parsed + if ( + caller_partition == target_partition + and caller_account == target_account + and caller_role == target_role + ): + verbose_logger.debug( + "Current identity already matches target role: %s", + aws_role_name, + ) + return True + + except Exception as e: + verbose_logger.debug( + "Could not determine current role identity: %s", str(e) + ) + + return False + + @tracer.wrap() + def _auth_with_web_identity_token( + self, + aws_web_identity_token: str, + aws_role_name: str, + aws_session_name: str, + aws_region_name: Optional[str], + aws_sts_endpoint: Optional[str], + aws_external_id: Optional[str] = None, + ssl_verify: Optional[Union[bool, str]] = None, + ) -> Tuple[Credentials, Optional[int]]: + """ + Authenticate with AWS Web Identity Token + """ + import boto3 + + verbose_logger.debug( + f"IN Web Identity Token: {aws_web_identity_token} | Role Name: {aws_role_name} | Session Name: {aws_session_name}" + ) + + if aws_sts_endpoint is None: + sts_endpoint = f"https://sts.{aws_region_name}.amazonaws.com" + else: + sts_endpoint = aws_sts_endpoint + + oidc_token = get_secret(aws_web_identity_token) + + if oidc_token is None: + raise AwsAuthError( + message="OIDC token could not be retrieved from secret manager.", + status_code=401, + ) + + with tracer.trace("boto3.client(sts)"): + sts_client = boto3.client( + "sts", + region_name=aws_region_name, + endpoint_url=sts_endpoint, + verify=self._get_ssl_verify(ssl_verify), + ) + + # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html + assume_role_params = { + "RoleArn": aws_role_name, + "RoleSessionName": aws_session_name, + "WebIdentityToken": oidc_token, + "DurationSeconds": 3600, + "Policy": '{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringLike":{"aws:UserAgent":"litellm/*"}}}]}', + } + + # Add ExternalId parameter if provided + if aws_external_id is not None: + assume_role_params["ExternalId"] = aws_external_id + + sts_response = sts_client.assume_role_with_web_identity(**assume_role_params) + + iam_creds_dict = { + "aws_access_key_id": sts_response["Credentials"]["AccessKeyId"], + "aws_secret_access_key": sts_response["Credentials"]["SecretAccessKey"], + "aws_session_token": sts_response["Credentials"]["SessionToken"], + "region_name": aws_region_name, + } + + if sts_response["PackedPolicySize"] > BEDROCK_MAX_POLICY_SIZE: + verbose_logger.warning( + f"The policy size is greater than 75% of the allowed size, PackedPolicySize: {sts_response['PackedPolicySize']}" + ) + + with tracer.trace("boto3.Session(**iam_creds_dict)"): + session = boto3.Session(**iam_creds_dict) + + iam_creds = session.get_credentials() + return iam_creds, self._get_default_ttl_for_boto3_credentials() + + def _handle_irsa_cross_account( + self, + irsa_role_arn: str, + aws_role_name: str, + aws_session_name: str, + region: str, + web_identity_token_file: str, + aws_external_id: Optional[str] = None, + aws_sts_endpoint: Optional[str] = None, + ssl_verify: Optional[Union[bool, str]] = None, + ) -> dict: + """Handle cross-account role assumption for IRSA.""" + import boto3 + + verbose_logger.debug("Cross-account role assumption detected") + + # Read the web identity token + with open(web_identity_token_file, "r") as f: + web_identity_token = f.read().strip() + + irsa_sts_kwargs: dict = { + "region_name": region, + "verify": self._get_ssl_verify(ssl_verify), + } + if aws_sts_endpoint is not None: + irsa_sts_kwargs["endpoint_url"] = aws_sts_endpoint + + # Create an STS client without credentials + with tracer.trace("boto3.client(sts) for manual IRSA"): + sts_client = boto3.client("sts", **irsa_sts_kwargs) + + # Manually assume the IRSA role with the session name + verbose_logger.debug( + f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}" + ) + irsa_response = sts_client.assume_role_with_web_identity( + RoleArn=irsa_role_arn, + RoleSessionName=aws_session_name, + WebIdentityToken=web_identity_token, + ) + + # Extract the credentials from the IRSA assumption + irsa_creds = irsa_response["Credentials"] + + # Create a new STS client with the IRSA credentials + with tracer.trace("boto3.client(sts) with manual IRSA credentials"): + sts_client_with_creds = boto3.client( + "sts", + aws_access_key_id=irsa_creds["AccessKeyId"], + aws_secret_access_key=irsa_creds["SecretAccessKey"], + aws_session_token=irsa_creds["SessionToken"], + **irsa_sts_kwargs, + ) + + # Get current caller identity for debugging + try: + caller_identity = sts_client_with_creds.get_caller_identity() + verbose_logger.debug( + f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}" + ) + except Exception as e: + verbose_logger.debug(f"Failed to get caller identity: {e}") + + # Now assume the target role + verbose_logger.debug( + f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}" + ) + assume_role_params = { + "RoleArn": aws_role_name, + "RoleSessionName": aws_session_name, + } + + # Add ExternalId parameter if provided + if aws_external_id is not None: + assume_role_params["ExternalId"] = aws_external_id + + return sts_client_with_creds.assume_role(**assume_role_params) + + def _handle_irsa_same_account( + self, + aws_role_name: str, + aws_session_name: str, + region: str, + aws_external_id: Optional[str] = None, + aws_sts_endpoint: Optional[str] = None, + ssl_verify: Optional[Union[bool, str]] = None, + ) -> dict: + """Handle same-account role assumption for IRSA.""" + import boto3 + + irsa_sts_kwargs: dict = { + "region_name": region, + "verify": self._get_ssl_verify(ssl_verify), + } + if aws_sts_endpoint is not None: + irsa_sts_kwargs["endpoint_url"] = aws_sts_endpoint + + verbose_logger.debug("Same account role assumption, using automatic IRSA") + with tracer.trace("boto3.client(sts) with automatic IRSA"): + sts_client = boto3.client("sts", **irsa_sts_kwargs) + + # Get current caller identity for debugging + try: + caller_identity = sts_client.get_caller_identity() + verbose_logger.debug( + f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}" + ) + except Exception as e: + verbose_logger.debug(f"Failed to get caller identity: {e}") + + # Assume the role + verbose_logger.debug( + f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}" + ) + assume_role_params = { + "RoleArn": aws_role_name, + "RoleSessionName": aws_session_name, + } + + # Add ExternalId parameter if provided + if aws_external_id is not None: + assume_role_params["ExternalId"] = aws_external_id + + return sts_client.assume_role(**assume_role_params) + + def _extract_credentials_and_ttl( + self, sts_response: dict + ) -> Tuple[Credentials, Optional[int]]: + """Extract credentials and TTL from STS response.""" + from botocore.credentials import Credentials + + sts_credentials = sts_response["Credentials"] + credentials = Credentials( + access_key=sts_credentials["AccessKeyId"], + secret_key=sts_credentials["SecretAccessKey"], + token=sts_credentials["SessionToken"], + ) + + expiration_time = sts_credentials["Expiration"] + ttl = int( + (expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds() + ) + + return credentials, ttl + + @tracer.wrap() + def _auth_with_aws_role( + self, + aws_access_key_id: Optional[str], + aws_secret_access_key: Optional[str], + aws_session_token: Optional[str], + aws_role_name: str, + aws_session_name: str, + aws_region_name: Optional[str] = None, + aws_sts_endpoint: Optional[str] = None, + aws_external_id: Optional[str] = None, + ssl_verify: Optional[Union[bool, str]] = None, + ) -> Tuple[Credentials, Optional[int]]: + """ + Authenticate with AWS Role + """ + import boto3 + from botocore.credentials import Credentials + + # Check if we're in an EKS/IRSA environment + web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE") + irsa_role_arn = os.getenv("AWS_ROLE_ARN") + + region = ( + aws_region_name + or os.getenv("AWS_REGION") + or os.getenv("AWS_DEFAULT_REGION") + ) + + # If we have IRSA environment variables and no explicit credentials, + # we need to use the web identity token flow + if ( + web_identity_token_file + and irsa_role_arn + and aws_access_key_id is None + and aws_secret_access_key is None + ): + # For cross-account role assumption with specific session names, + # we need to manually assume the IRSA role first with the correct session name + verbose_logger.debug( + f"IRSA detected: using web identity token from {web_identity_token_file}" + ) + + try: + # Use passed-in region when set, else env, else default (align with AssumeRole path) + region = region or "us-east-1" + + # Check if we need to do cross-account role assumption + if aws_role_name != irsa_role_arn: + sts_response = self._handle_irsa_cross_account( + irsa_role_arn, + aws_role_name, + aws_session_name, + region, + web_identity_token_file, + aws_external_id, + aws_sts_endpoint=aws_sts_endpoint, + ssl_verify=ssl_verify, + ) + else: + sts_response = self._handle_irsa_same_account( + aws_role_name, + aws_session_name, + region, + aws_external_id, + aws_sts_endpoint=aws_sts_endpoint, + ssl_verify=ssl_verify, + ) + + return self._extract_credentials_and_ttl(sts_response) + + except Exception as e: + verbose_logger.debug(f"Failed to assume role via IRSA: {e}") + if "AccessDenied" in str( + e + ) and "is not authorized to perform: sts:AssumeRole" in str(e): + # Provide a more helpful error message for trust policy issues + verbose_logger.error( + f"Access denied when trying to assume role {aws_role_name}. " + f"Please ensure the trust policy of {aws_role_name} allows " + f"the current role to assume it. Current identity: check logs with verbose mode." + ) + # Re-raise the exception instead of falling through + raise + + # In EKS/IRSA environments, use ambient credentials (no explicit keys needed) + # This allows the web identity token to work automatically + sts_client_kwargs: dict = {"verify": self._get_ssl_verify(ssl_verify)} + if region is not None: + sts_client_kwargs["region_name"] = region + if aws_sts_endpoint is not None: + sts_client_kwargs["endpoint_url"] = aws_sts_endpoint + if aws_access_key_id is None and aws_secret_access_key is None: + with tracer.trace("boto3.client(sts)"): + sts_client = boto3.client("sts", **sts_client_kwargs) + else: + with tracer.trace("boto3.client(sts)"): + sts_client = boto3.client( + "sts", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + **sts_client_kwargs, + ) + + assume_role_params = { + "RoleArn": aws_role_name, + "RoleSessionName": aws_session_name, + } + + # Add ExternalId parameter if provided + if aws_external_id is not None: + assume_role_params["ExternalId"] = aws_external_id + + try: + sts_response = sts_client.assume_role(**assume_role_params) + except Exception as e: + error_str = str(e) + if "AccessDenied" in error_str: + # Only fall back to ambient credentials if we can positively + # confirm the caller is already the target role (same account, + # partition, and role name). This avoids silently using the + # wrong identity when there is a genuine trust-policy or + # permission misconfiguration. + if self._is_already_running_as_role( + aws_role_name, ssl_verify=ssl_verify + ): + verbose_logger.warning( + "AssumeRole failed for %s (%s). " + "Caller is already running as this role; " + "falling back to ambient credentials.", + aws_role_name, + error_str, + ) + return self._auth_with_env_vars() + # Genuine permission error — re-raise + verbose_logger.error( + "AssumeRole AccessDenied for %s and caller is NOT " + "the same role. Re-raising. Error: %s", + aws_role_name, + error_str, + ) + raise + + # Extract the credentials from the response and convert to Session Credentials + sts_credentials = sts_response["Credentials"] + credentials = Credentials( + access_key=sts_credentials["AccessKeyId"], + secret_key=sts_credentials["SecretAccessKey"], + token=sts_credentials["SessionToken"], + ) + + sts_expiry = sts_credentials["Expiration"] + # Convert to timezone-aware datetime for comparison + current_time = datetime.now(sts_expiry.tzinfo) + sts_ttl = (sts_expiry - current_time).total_seconds() - 60 + return credentials, sts_ttl + + @tracer.wrap() + def _auth_with_aws_profile( + self, aws_profile_name: str + ) -> Tuple[Credentials, Optional[int]]: + """ + Authenticate with AWS profile + """ + import boto3 + + # uses auth values from AWS profile usually stored in ~/.aws/credentials + with tracer.trace("boto3.Session(profile_name=aws_profile_name)"): + client = boto3.Session(profile_name=aws_profile_name) + return client.get_credentials(), None + + @tracer.wrap() + def _auth_with_aws_session_token( + self, + aws_access_key_id: str, + aws_secret_access_key: str, + aws_session_token: str, + ) -> Tuple[Credentials, Optional[int]]: + """ + Authenticate with AWS Session Token + """ + ### CHECK FOR AWS SESSION TOKEN ### + from botocore.credentials import Credentials + + credentials = Credentials( + access_key=aws_access_key_id, + secret_key=aws_secret_access_key, + token=aws_session_token, + ) + + return credentials, None + + @tracer.wrap() + def _auth_with_access_key_and_secret_key( + self, + aws_access_key_id: str, + aws_secret_access_key: str, + aws_region_name: Optional[str], + ) -> Tuple[Credentials, Optional[int]]: + """ + Authenticate with AWS Access Key and Secret Key + """ + import boto3 + + # Check if credentials are already in cache. These credentials have no expiry time. + with tracer.trace( + "boto3.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name)" + ): + session = boto3.Session( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=aws_region_name, + ) + + credentials = session.get_credentials() + return credentials, self._get_default_ttl_for_boto3_credentials() + + @tracer.wrap() + def _auth_with_env_vars(self) -> Tuple[Credentials, Optional[int]]: + """ + Authenticate with AWS Environment Variables + """ + import boto3 + + with tracer.trace("boto3.Session()"): + session = boto3.Session() + credentials = session.get_credentials() + return credentials, None + + @tracer.wrap() + def _get_default_ttl_for_boto3_credentials(self) -> int: + """ + Get the default TTL for boto3 credentials + + Returns `3600-60` which is 59 minutes + """ + return 3600 - 60 + + def get_runtime_endpoint( + self, + api_base: Optional[str], + aws_bedrock_runtime_endpoint: Optional[str], + aws_region_name: str, + endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]] = "runtime", + ) -> Tuple[str, str]: + env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT") + if api_base is not None: + endpoint_url = api_base + elif aws_bedrock_runtime_endpoint is not None and isinstance( + aws_bedrock_runtime_endpoint, str + ): + endpoint_url = aws_bedrock_runtime_endpoint + elif env_aws_bedrock_runtime_endpoint and isinstance( + env_aws_bedrock_runtime_endpoint, str + ): + endpoint_url = env_aws_bedrock_runtime_endpoint + else: + endpoint_url = self._select_default_endpoint_url( + endpoint_type=endpoint_type, + aws_region_name=aws_region_name, + ) + + # Determine proxy_endpoint_url + if aws_bedrock_runtime_endpoint is not None and isinstance( + aws_bedrock_runtime_endpoint, str + ): + proxy_endpoint_url = aws_bedrock_runtime_endpoint + elif env_aws_bedrock_runtime_endpoint and isinstance( + env_aws_bedrock_runtime_endpoint, str + ): + proxy_endpoint_url = env_aws_bedrock_runtime_endpoint + else: + proxy_endpoint_url = endpoint_url + + return endpoint_url, proxy_endpoint_url + + def _select_default_endpoint_url( + self, + endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]], + aws_region_name: str, + ) -> str: + """ + Select the default endpoint url based on the endpoint type + + Default endpoint url is https://bedrock-runtime.{aws_region_name}.amazonaws.com + """ + if endpoint_type == "agent": + return f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com" + elif endpoint_type == "agentcore": + return f"https://bedrock-agentcore.{aws_region_name}.amazonaws.com" + else: + return f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" + + def _get_boto_credentials_from_optional_params( + self, optional_params: dict, model: Optional[str] = None + ) -> Boto3CredentialsInfo: + """ + Get boto3 credentials from optional params + + Args: + optional_params (dict): Optional parameters for the model call + + Returns: + Credentials: Boto3 credentials object + """ + try: + from botocore.credentials import Credentials + except ImportError: + raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + ## CREDENTIALS ## + # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them + aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) + aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_session_token = optional_params.pop("aws_session_token", None) + aws_region_name = self._get_aws_region_name(optional_params, model) + optional_params.pop("aws_region_name", None) + aws_role_name = optional_params.pop("aws_role_name", None) + aws_session_name = optional_params.pop("aws_session_name", None) + aws_profile_name = optional_params.pop("aws_profile_name", None) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) + aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None) + aws_bedrock_runtime_endpoint = optional_params.pop( + "aws_bedrock_runtime_endpoint", None + ) # https://bedrock-runtime.{region_name}.amazonaws.com + aws_external_id = optional_params.pop("aws_external_id", None) + + credentials: Credentials = self.get_credentials( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region_name=aws_region_name, + aws_session_name=aws_session_name, + aws_profile_name=aws_profile_name, + aws_role_name=aws_role_name, + aws_web_identity_token=aws_web_identity_token, + aws_sts_endpoint=aws_sts_endpoint, + aws_external_id=aws_external_id, + ) + + return Boto3CredentialsInfo( + credentials=credentials, + aws_region_name=aws_region_name, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + ) + + @tracer.wrap() + def get_request_headers( + self, + credentials: Credentials, + aws_region_name: str, + extra_headers: Optional[dict], + endpoint_url: str, + data: Union[str, bytes], + headers: dict, + api_key: Optional[str] = None, + ) -> AWSPreparedRequest: + if api_key is not None: + aws_bearer_token: Optional[str] = api_key + else: + aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK") + + if aws_bearer_token: + try: + from botocore.awsrequest import AWSRequest + except ImportError: + raise ImportError( + "Missing boto3 to call bedrock. Run 'pip install boto3'." + ) + headers["Authorization"] = f"Bearer {aws_bearer_token}" + request = AWSRequest( + method="POST", url=endpoint_url, data=data, headers=headers + ) + else: + try: + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + except ImportError: + raise ImportError( + "Missing boto3 to call bedrock. Run 'pip install boto3'." + ) + + # Filter headers for AWS signature calculation + # AWS SigV4 only includes specific headers in signature calculation + aws_signature_headers = self._filter_headers_for_aws_signature(headers) + sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name) + request = AWSRequest( + method="POST", + url=endpoint_url, + data=data, + headers=aws_signature_headers, + ) + sigv4.add_auth(request) + + # Add back all original headers (including forwarded ones) after signature calculation + for header_name, header_value in headers.items(): + if header_value is not None: + request.headers[header_name] = header_value + + if ( + extra_headers is not None and "Authorization" in extra_headers + ): # prevent sigv4 from overwriting the auth header + request.headers["Authorization"] = extra_headers["Authorization"] + prepped = request.prepare() + + return prepped + + def _filter_headers_for_aws_signature(self, headers: dict) -> dict: + """ + Filter headers to only include those that AWS SigV4 includes in signature calculation. + This Fixes forwarded client headers from breaking the signature calculation. + """ + aws_signature_headers = {} + aws_headers = { + "host", + "content-type", + "date", + "x-amz-date", + "x-amz-security-token", + "x-amz-content-sha256", + "x-amz-algorithm", + "x-amz-credential", + "x-amz-signedheaders", + "x-amz-signature", + } + + for header_name, header_value in headers.items(): + if header_value is None: + continue + header_lower = header_name.lower() + if ( + header_lower in aws_headers + or header_lower.startswith("x-amz-") + or header_lower.startswith("x-amzn-") + ): + aws_signature_headers[header_name] = header_value + + return aws_signature_headers + + def _sign_request( + self, + service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"], + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + api_key: Optional[str] = None, + ) -> Tuple[dict, Optional[bytes]]: + """ + Sign a request for Bedrock or Sagemaker + + Returns: + Tuple[dict, Optional[str]]: A tuple containing the headers and the json str body of the request + """ + if api_key is not None: + aws_bearer_token: Optional[str] = api_key + else: + aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK") + + # If aws bearer token is set, use it directly in the header + if aws_bearer_token: + headers = headers or {} + headers["Content-Type"] = "application/json" + headers["Authorization"] = f"Bearer {aws_bearer_token}" + return headers, json.dumps(request_data).encode() + + # If no bearer token is set, proceed with the existing SigV4 authentication + try: + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + except ImportError: + raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + + ## CREDENTIALS ## + # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them + aws_secret_access_key = optional_params.get("aws_secret_access_key", None) + aws_access_key_id = optional_params.get("aws_access_key_id", None) + aws_session_token = optional_params.get("aws_session_token", None) + aws_role_name = optional_params.get("aws_role_name", None) + aws_session_name = optional_params.get("aws_session_name", None) + aws_profile_name = optional_params.get("aws_profile_name", None) + aws_web_identity_token = optional_params.get("aws_web_identity_token", None) + aws_sts_endpoint = optional_params.get("aws_sts_endpoint", None) + aws_external_id = optional_params.get("aws_external_id", None) + aws_region_name = self._get_aws_region_name( + optional_params=optional_params, model=model + ) + + credentials: Credentials = self.get_credentials( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region_name=aws_region_name, + aws_session_name=aws_session_name, + aws_profile_name=aws_profile_name, + aws_role_name=aws_role_name, + aws_web_identity_token=aws_web_identity_token, + aws_sts_endpoint=aws_sts_endpoint, + aws_external_id=aws_external_id, + ) + + sigv4 = SigV4Auth(credentials, service_name, aws_region_name) + if headers is not None: + headers = {"Content-Type": "application/json", **headers} + else: + headers = {"Content-Type": "application/json"} + + aws_signature_headers = self._filter_headers_for_aws_signature(headers) + request = AWSRequest( + method="POST", + url=api_base, + data=json.dumps(request_data), + headers=aws_signature_headers, + ) + sigv4.add_auth(request) + + request_headers_dict = dict(request.headers) + # Add back original headers after signing. Only headers in SignedHeaders + # are integrity-protected; forwarded headers (x-forwarded-*) must remain unsigned. + for header_name, header_value in headers.items(): + if header_value is not None: + request_headers_dict[header_name] = header_value + if ( + headers is not None and "Authorization" in headers + ): # prevent sigv4 from overwriting the auth header + request_headers_dict["Authorization"] = headers["Authorization"] + + return request_headers_dict, request.body diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/handler.py new file mode 100644 index 0000000000000000000000000000000000000000..e0c7c08836245df486457a568a392740986b03cb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/handler.py @@ -0,0 +1,97 @@ +from openai.types.batch import BatchRequestCounts +from openai.types.batch import Metadata as OpenAIBatchMetadata + +from litellm.types.utils import LiteLLMBatch + + +class BedrockBatchesHandler: + """ + Handler for Bedrock Batches. + + Specific providers/models needed some special handling. + + E.g. Twelve Labs Embedding Async Invoke + """ + + @staticmethod + def _handle_async_invoke_status( + batch_id: str, aws_region_name: str, logging_obj=None, **kwargs + ) -> "LiteLLMBatch": + """ + Handle async invoke status check for AWS Bedrock. + + This is for Twelve Labs Embedding Async Invoke. + + Args: + batch_id: The async invoke ARN + aws_region_name: AWS region name + **kwargs: Additional parameters + + Returns: + dict: Status information including status, output_file_id (S3 URL), etc. + """ + import asyncio + + from litellm.llms.bedrock.embed.embedding import BedrockEmbedding + + async def _async_get_status(): + # Create embedding handler instance + embedding_handler = BedrockEmbedding() + + # Get the status of the async invoke job + status_response = await embedding_handler._get_async_invoke_status( + invocation_arn=batch_id, + aws_region_name=aws_region_name, + logging_obj=logging_obj, + **kwargs, + ) + + # Transform response to a LiteLLMBatch object + from litellm.types.utils import LiteLLMBatch + + openai_batch_metadata: OpenAIBatchMetadata = { + "output_file_id": status_response["outputDataConfig"][ + "s3OutputDataConfig" + ]["s3Uri"], + "failure_message": status_response.get("failureMessage") or "", + "model_arn": status_response["modelArn"], + } + + result = LiteLLMBatch( + id=status_response["invocationArn"], + object="batch", + status=status_response["status"], + created_at=status_response["submitTime"], + in_progress_at=status_response["lastModifiedTime"], + completed_at=status_response.get("endTime"), + failed_at=status_response.get("endTime") + if status_response["status"] == "failed" + else None, + request_counts=BatchRequestCounts( + total=1, + completed=1 if status_response["status"] == "completed" else 0, + failed=1 if status_response["status"] == "failed" else 0, + ), + metadata=openai_batch_metadata, + completion_window="24h", + endpoint="/v1/embeddings", + input_file_id="", + ) + + return result + + # Since this function is called from within an async context via run_in_executor, + # we need to create a new event loop in a thread to avoid conflicts + import concurrent.futures + + def run_in_thread(): + new_loop = asyncio.new_event_loop() + asyncio.set_event_loop(new_loop) + try: + return new_loop.run_until_complete(_async_get_status()) + finally: + new_loop.close() + + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(run_in_thread) + return future.result() diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..5d008038ca9c3868017623b139fb463da5c6ac6a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/batches/transformation.py @@ -0,0 +1,549 @@ +import os +import time +from typing import Any, Dict, List, Literal, Optional, Union, cast + +from httpx import Headers, Response + +from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.bedrock import ( + BedrockCreateBatchRequest, + BedrockCreateBatchResponse, + BedrockInputDataConfig, + BedrockOutputDataConfig, + BedrockS3InputDataConfig, + BedrockS3OutputDataConfig, +) +from litellm.types.llms.openai import ( + AllMessageValues, + CreateBatchRequest, +) +from litellm.types.utils import LiteLLMBatch, LlmProviders + +from ..base_aws_llm import BaseAWSLLM +from ..common_utils import CommonBatchFilesUtils + + +class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig): + """ + Config for Bedrock Batches - handles batch job creation and management for Bedrock + """ + + def __init__(self): + super().__init__() + self.common_utils = CommonBatchFilesUtils() + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BEDROCK + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate and prepare environment for Bedrock batch requests. + AWS credentials are handled by BaseAWSLLM. + """ + # Add any Bedrock-specific headers if needed + return headers + + def get_complete_batch_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: Dict, + litellm_params: Dict, + data: CreateBatchRequest, + ) -> str: + """ + Get the complete URL for Bedrock batch creation. + Bedrock batch jobs are created via the model invocation job API. + """ + aws_region_name = self._get_aws_region_name(optional_params, model) + + # Bedrock model invocation job endpoint + # Format: https://bedrock.{region}.amazonaws.com/model-invocation-job + bedrock_endpoint = ( + f"https://bedrock.{aws_region_name}.amazonaws.com/model-invocation-job" + ) + + return bedrock_endpoint + + def transform_create_batch_request( + self, + model: str, + create_batch_data: CreateBatchRequest, + optional_params: dict, + litellm_params: dict, + ) -> Dict[str, Any]: + """ + Transform the batch creation request to Bedrock format. + + Bedrock batch inference requires: + - modelId: The Bedrock model ID + - jobName: Unique name for the batch job + - inputDataConfig: Configuration for input data (S3 location) + - outputDataConfig: Configuration for output data (S3 location) + - roleArn: IAM role ARN for the batch job + """ + # Get required parameters + input_file_id = create_batch_data.get("input_file_id") + if not input_file_id: + raise ValueError("input_file_id is required for Bedrock batch creation") + + # Extract S3 information from file ID using common utility + input_bucket, input_key = self.common_utils.parse_s3_uri(input_file_id) + + # Get output S3 configuration + output_bucket = litellm_params.get("s3_output_bucket_name") or os.getenv( + "AWS_S3_OUTPUT_BUCKET_NAME" + ) + if not output_bucket: + # Use same bucket as input if no output bucket specified + output_bucket = input_bucket + + # Get IAM role ARN + role_arn = ( + litellm_params.get("aws_batch_role_arn") + or optional_params.get("aws_batch_role_arn") + or os.getenv("AWS_BATCH_ROLE_ARN") + ) + if not role_arn: + raise ValueError( + "AWS IAM role ARN is required for Bedrock batch jobs. " + "Set 'aws_batch_role_arn' in litellm_params or AWS_BATCH_ROLE_ARN env var" + ) + + if not model: + raise ValueError( + "Could not determine Bedrock model ID. Please pass `model` in your request body." + ) + + # Generate job name with the correct model ID using common utility + job_name = self.common_utils.generate_unique_job_name(model, prefix="litellm") + output_key = f"litellm-batch-outputs/{job_name}/" + + # Build input data config + input_data_config: BedrockInputDataConfig = { + "s3InputDataConfig": BedrockS3InputDataConfig( + s3Uri=f"s3://{input_bucket}/{input_key}" + ) + } + + # Build output data config + s3_output_config: BedrockS3OutputDataConfig = BedrockS3OutputDataConfig( + s3Uri=f"s3://{output_bucket}/{output_key}" + ) + + # Add optional KMS encryption key ID if provided + s3_encryption_key_id = litellm_params.get( + "s3_encryption_key_id" + ) or get_secret_str("AWS_S3_ENCRYPTION_KEY_ID") + if s3_encryption_key_id: + s3_output_config["s3EncryptionKeyId"] = s3_encryption_key_id + + output_data_config: BedrockOutputDataConfig = { + "s3OutputDataConfig": s3_output_config + } + + # Create Bedrock batch request with proper typing + bedrock_request: BedrockCreateBatchRequest = { + "modelId": model, + "jobName": job_name, + "inputDataConfig": input_data_config, + "outputDataConfig": output_data_config, + "roleArn": role_arn, + } + + # Add optional parameters if provided + completion_window = create_batch_data.get("completion_window") + if completion_window: + # Map OpenAI completion window to Bedrock timeout + # OpenAI uses "24h", Bedrock expects timeout in hours + if completion_window == "24h": + bedrock_request["timeoutDurationInHours"] = 24 + + # For Bedrock, we need to return a pre-signed request with AWS auth headers + # Use common utility for AWS signing + endpoint_url = f"https://bedrock.{self._get_aws_region_name(optional_params, model)}.amazonaws.com/model-invocation-job" + signed_headers, signed_data = self.common_utils.sign_aws_request( + service_name="bedrock", + data=bedrock_request, + endpoint_url=endpoint_url, + optional_params=optional_params, + method="POST", + ) + + # Return a pre-signed request format that the HTTP handler can use + return { + "method": "POST", + "url": endpoint_url, + "headers": signed_headers, + "data": signed_data.decode("utf-8"), + } + + def transform_create_batch_response( + self, + model: Optional[str], + raw_response: Response, + logging_obj: Any, + litellm_params: dict, + ) -> LiteLLMBatch: + """ + Transform Bedrock batch creation response to LiteLLM format. + """ + try: + response_data: BedrockCreateBatchResponse = raw_response.json() + except Exception as e: + raise ValueError(f"Failed to parse Bedrock batch response: {e}") + + # Extract information from typed Bedrock response + job_arn = response_data.get("jobArn", "") + status_str: str = str(response_data.get("status", "Submitted")) + + # Map Bedrock status to OpenAI-compatible status + status_mapping: Dict[str, str] = { + "Submitted": "validating", + "Validating": "validating", + "Scheduled": "in_progress", + "InProgress": "in_progress", + "PartiallyCompleted": "completed", + "Completed": "completed", + "Failed": "failed", + "Stopping": "cancelling", + "Stopped": "cancelled", + "Expired": "expired", + } + + openai_status = cast( + Literal[ + "validating", + "failed", + "in_progress", + "finalizing", + "completed", + "expired", + "cancelling", + "cancelled", + ], + status_mapping.get(status_str, "validating"), + ) + + # Get original request data from litellm_params if available + original_request = litellm_params.get("original_batch_request", {}) + + # Create LiteLLM batch object + return LiteLLMBatch( + id=job_arn, # Use ARN as the batch ID + object="batch", + endpoint=original_request.get("endpoint", "/v1/chat/completions"), + errors=None, + input_file_id=original_request.get("input_file_id", ""), + completion_window=original_request.get("completion_window", "24h"), + status=openai_status, + output_file_id=None, # Will be populated when job completes + error_file_id=None, + created_at=int(time.time()), + in_progress_at=int(time.time()) if status_str == "InProgress" else None, + expires_at=None, + finalizing_at=None, + completed_at=None, + failed_at=None, + expired_at=None, + cancelling_at=None, + cancelled_at=None, + request_counts=None, + metadata=original_request.get("metadata", {}), + ) + + def transform_retrieve_batch_request( + self, + batch_id: str, + optional_params: dict, + litellm_params: dict, + ) -> Dict[str, Any]: + """ + Transform batch retrieval request for Bedrock. + + Args: + batch_id: Bedrock job ARN + optional_params: Optional parameters + litellm_params: LiteLLM parameters + + Returns: + Transformed request data for Bedrock GetModelInvocationJob API + """ + # For Bedrock, batch_id should be the full job ARN + # The GetModelInvocationJob API expects the full ARN as the identifier + if not batch_id.startswith("arn:aws:bedrock:"): + raise ValueError(f"Invalid batch_id format. Expected ARN, got: {batch_id}") + + # Extract the job identifier from the ARN - use the full ARN path part + # ARN format: arn:aws:bedrock:region:account:model-invocation-job/job-name + arn_parts = batch_id.split(":") + if len(arn_parts) < 6: + raise ValueError(f"Invalid ARN format: {batch_id}") + + region = arn_parts[3] + # arn_parts[5] contains "model-invocation-job/{jobId}" + + # Build the endpoint URL for GetModelInvocationJob + # AWS API format: GET /model-invocation-job/{jobIdentifier} + # Use the FULL ARN as jobIdentifier and URL-encode it (includes ':' and '/') + import urllib.parse as _ul + + encoded_arn = _ul.quote(batch_id, safe="") + endpoint_url = ( + f"https://bedrock.{region}.amazonaws.com/model-invocation-job/{encoded_arn}" + ) + + # Use common utility for AWS signing + signed_headers, _ = self.common_utils.sign_aws_request( + service_name="bedrock", + data={}, # GET request has no body + endpoint_url=endpoint_url, + optional_params=optional_params, + method="GET", + ) + + # Return pre-signed request format + return { + "method": "GET", + "url": endpoint_url, + "headers": signed_headers, + "data": None, + } + + def _parse_timestamps_and_status(self, response_data, status_str: str): + """Helper to parse timestamps based on status.""" + import datetime + + def parse_timestamp(ts_str: Optional[str]) -> Optional[int]: + if not ts_str: + return None + try: + dt = datetime.datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + return int(dt.timestamp()) + except Exception: + return None + + created_at = parse_timestamp( + str(response_data.get("submitTime")) + if response_data.get("submitTime") is not None + else None + ) + in_progress_states = {"InProgress", "Validating", "Scheduled"} + in_progress_at = ( + parse_timestamp( + str(response_data.get("lastModifiedTime")) + if response_data.get("lastModifiedTime") is not None + else None + ) + if status_str in in_progress_states + else None + ) + completed_at = ( + parse_timestamp( + str(response_data.get("endTime")) + if response_data.get("endTime") is not None + else None + ) + if status_str in {"Completed", "PartiallyCompleted"} + else None + ) + failed_at = ( + parse_timestamp( + str(response_data.get("endTime")) + if response_data.get("endTime") is not None + else None + ) + if status_str == "Failed" + else None + ) + cancelled_at = ( + parse_timestamp( + str(response_data.get("endTime")) + if response_data.get("endTime") is not None + else None + ) + if status_str == "Stopped" + else None + ) + expires_at = parse_timestamp( + str(response_data.get("jobExpirationTime")) + if response_data.get("jobExpirationTime") is not None + else None + ) + + return ( + created_at, + in_progress_at, + completed_at, + failed_at, + cancelled_at, + expires_at, + ) + + def _extract_file_configs(self, response_data): + """Helper to extract input and output file configurations.""" + # Extract input file ID + input_file_id = "" + input_data_config = response_data.get("inputDataConfig", {}) + if isinstance(input_data_config, dict): + s3_input_config = input_data_config.get("s3InputDataConfig", {}) + if isinstance(s3_input_config, dict): + input_file_id = s3_input_config.get("s3Uri", "") + + # Extract output file ID + output_file_id = None + output_data_config = response_data.get("outputDataConfig", {}) + if isinstance(output_data_config, dict): + s3_output_config = output_data_config.get("s3OutputDataConfig", {}) + if isinstance(s3_output_config, dict): + output_file_id = s3_output_config.get("s3Uri", "") + + return input_file_id, output_file_id + + def _extract_errors_and_metadata(self, response_data, raw_response): + """Helper to extract errors and enriched metadata.""" + # Extract errors + message = response_data.get("message") + errors = None + if message: + from openai.types.batch import Errors + from openai.types.batch_error import BatchError + + errors = Errors( + data=[BatchError(message=message, code=str(raw_response.status_code))], + object="list", + ) + + # Enrich metadata with useful Bedrock fields + enriched_metadata_raw: Dict[str, Any] = { + "jobName": response_data.get("jobName"), + "clientRequestToken": response_data.get("clientRequestToken"), + "modelId": response_data.get("modelId"), + "roleArn": response_data.get("roleArn"), + "timeoutDurationInHours": response_data.get("timeoutDurationInHours"), + "vpcConfig": response_data.get("vpcConfig"), + } + import json as _json + + enriched_metadata: Dict[str, str] = {} + for _k, _v in enriched_metadata_raw.items(): + if _v is None: + continue + if isinstance(_v, (dict, list)): + try: + enriched_metadata[_k] = _json.dumps(_v) + except Exception: + enriched_metadata[_k] = str(_v) + else: + enriched_metadata[_k] = str(_v) + + return errors, enriched_metadata + + def transform_retrieve_batch_response( + self, + model: Optional[str], + raw_response: Response, + logging_obj: Any, + litellm_params: dict, + ) -> LiteLLMBatch: + """ + Transform Bedrock batch retrieval response to LiteLLM format. + """ + from litellm.types.llms.bedrock import BedrockGetBatchResponse + + try: + response_data: BedrockGetBatchResponse = raw_response.json() + except Exception as e: + raise ValueError(f"Failed to parse Bedrock batch response: {e}") + + job_arn = response_data.get("jobArn", "") + status_str: str = str(response_data.get("status", "Submitted")) + + # Map Bedrock status to OpenAI-compatible status + status_mapping: Dict[str, str] = { + "Submitted": "validating", + "Validating": "validating", + "Scheduled": "in_progress", + "InProgress": "in_progress", + "PartiallyCompleted": "completed", + "Completed": "completed", + "Failed": "failed", + "Stopping": "cancelling", + "Stopped": "cancelled", + "Expired": "expired", + } + openai_status = cast( + Literal[ + "validating", + "failed", + "in_progress", + "finalizing", + "completed", + "expired", + "cancelling", + "cancelled", + ], + status_mapping.get(status_str, "validating"), + ) + + # Parse timestamps + ( + created_at, + in_progress_at, + completed_at, + failed_at, + cancelled_at, + expires_at, + ) = self._parse_timestamps_and_status(response_data, status_str) + + # Extract file configurations + input_file_id, output_file_id = self._extract_file_configs(response_data) + + # Extract errors and metadata + errors, enriched_metadata = self._extract_errors_and_metadata( + response_data, raw_response + ) + + return LiteLLMBatch( + id=job_arn, + object="batch", + endpoint="/v1/chat/completions", + errors=errors, + input_file_id=input_file_id, + completion_window="24h", + status=openai_status, + output_file_id=output_file_id, + error_file_id=None, + created_at=created_at or int(time.time()), + in_progress_at=in_progress_at, + expires_at=expires_at, + finalizing_at=None, + completed_at=completed_at, + failed_at=failed_at, + expired_at=None, + cancelling_at=None, + cancelled_at=cancelled_at, + request_counts=None, + metadata=enriched_metadata, + ) + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[Dict, Headers] + ) -> BaseLLMException: + """ + Get Bedrock-specific error class using common utility. + """ + return self.common_utils.get_error_class(error_message, status_code, headers) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8cd0e94e68e3814fe12ada33b8a544008071edd8 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/__init__.py @@ -0,0 +1,30 @@ +from typing import Optional + +from .converse_handler import BedrockConverseLLM +from .invoke_handler import ( + AmazonAnthropicClaudeStreamDecoder, + AmazonDeepSeekR1StreamDecoder, + AWSEventStreamDecoder, + BedrockLLM, +) + + +def get_bedrock_event_stream_decoder( + invoke_provider: Optional[str], model: str, sync_stream: bool, json_mode: bool +): + if invoke_provider and invoke_provider == "anthropic": + decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder( + model=model, + sync_stream=sync_stream, + json_mode=json_mode, + ) + return decoder + elif invoke_provider and invoke_provider == "deepseek_r1": + decoder = AmazonDeepSeekR1StreamDecoder( + model=model, + sync_stream=sync_stream, + ) + return decoder + else: + decoder = AWSEventStreamDecoder(model=model) + return decoder diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2c83261fc927d704fd4891d6c4c7794651fd6f2d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/__init__.py @@ -0,0 +1,3 @@ +from .transformation import AmazonAgentCoreConfig + +__all__ = ["AmazonAgentCoreConfig"] diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..d6eb5a734c4bc7e301573d0140ff50d96d8dc139 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -0,0 +1,1037 @@ +""" +Transformation for Bedrock AgentCore + +https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html +""" + +import json +from collections.abc import AsyncGenerator +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast +from urllib.parse import quote + +import httpx + +from litellm._logging import verbose_logger +from litellm._uuid import uuid +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, +) +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.types.llms.bedrock_agentcore import ( + AgentCoreMessage, + AgentCoreParsedResponse, + AgentCoreUsage, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ( + Choices, + Delta, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, + Usage, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + HTTPHandler = Any + AsyncHTTPHandler = Any + + +class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): + def __init__(self, **kwargs): + BaseConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + def get_supported_openai_params(self, model: str) -> List[str]: + """ + Bedrock AgentCore has 0 OpenAI compatible params + """ + return [] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI params to AgentCore params + """ + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete url for the request + """ + ### SET RUNTIME ENDPOINT ### + aws_bedrock_runtime_endpoint = optional_params.get( + "aws_bedrock_runtime_endpoint", None + ) + + # Extract ARN from model string + agent_runtime_arn = self._get_agent_runtime_arn(model) + + # Parse ARN to get region + region = self._extract_region_from_arn(agent_runtime_arn) + + # Build the base endpoint URL for AgentCore + # Note: We don't use get_runtime_endpoint as AgentCore has its own endpoint structure + if aws_bedrock_runtime_endpoint: + base_url = aws_bedrock_runtime_endpoint + else: + base_url = f"https://bedrock-agentcore.{region}.amazonaws.com" + + # Based on boto3 client.invoke_agent_runtime, the path is: + # /runtimes/{URL-ENCODED-ARN}/invocations?qualifier= + encoded_arn = quote(agent_runtime_arn, safe="") + endpoint_url = f"{base_url}/runtimes/{encoded_arn}/invocations" + + # Add qualifier as query parameter if provided + if "qualifier" in optional_params: + endpoint_url = f"{endpoint_url}?qualifier={optional_params['qualifier']}" + + return endpoint_url + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + # Set Accept header required by MCP servers on AgentCore + # Per MCP spec (Streamable HTTP transport): client MUST include Accept header + # listing both application/json and text/event-stream as supported content types + headers["Accept"] = "application/json, text/event-stream" + + # Check if api_key (bearer token) is provided for Cognito authentication + # Priority: api_key parameter first, then optional_params + jwt_token = api_key or optional_params.get("api_key") + if jwt_token: + verbose_logger.debug( + f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..." + ) + headers["Content-Type"] = "application/json" + headers["Authorization"] = f"Bearer {jwt_token}" + # Return headers with bearer token and JSON-encoded body (not SigV4 signed) + return headers, json.dumps(request_data).encode() + + # Otherwise, use AWS SigV4 authentication + verbose_logger.debug("AgentCore: Using AWS SigV4 authentication (IAM)") + return self._sign_request( + service_name="bedrock-agentcore", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + model=model, + stream=stream, + fake_stream=fake_stream, + api_key=api_key, + ) + + def _get_agent_runtime_arn(self, model: str) -> str: + """ + Extract ARN from model string + model = "agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC" + returns: "arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC" + """ + parts = model.split("/", 1) + if len(parts) != 2 or parts[0] != "agentcore": + raise ValueError( + "Invalid model format. Expected format: 'model=bedrock/agentcore/arn:aws:bedrock-agentcore:region:account:runtime/runtime_id'" + ) + return parts[1] + + def _extract_region_from_arn(self, arn: str) -> str: + """ + Extract region from ARN + arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC + returns: us-west-2 + """ + parts = arn.split(":") + if len(parts) >= 4: + return parts[3] + raise ValueError(f"Invalid ARN format: {arn}") + + def _get_runtime_session_id(self, optional_params: dict) -> str: + """ + Get or generate runtime session ID (must be 33+ chars) + """ + session_id = optional_params.get("runtimeSessionId", None) + if session_id: + verbose_logger.debug(f"Using provided runtimeSessionId: {session_id}") + return session_id + + # Generate a session ID with 33+ characters + generated_id = f"litellm-session-{str(uuid.uuid4())}" + verbose_logger.debug(f"Generated new session ID: {generated_id}") + return generated_id + + def _get_runtime_user_id(self, optional_params: dict) -> Optional[str]: + """ + Get runtime user ID if provided + """ + user_id = optional_params.get("runtimeUserId", None) + if user_id: + verbose_logger.debug(f"Using provided runtimeUserId: {user_id}") + return user_id + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to AgentCore format. + + Based on boto3's implementation: + - Session ID goes in header: X-Amzn-Bedrock-AgentCore-Runtime-Session-Id + - User ID goes in header: X-Amzn-Bedrock-AgentCore-Runtime-User-Id + - Qualifier goes as query parameter + - Only the payload goes in the request body + + Returns: + dict: Payload dict containing the prompt + """ + verbose_logger.debug( + f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}" + ) + + # Use the last message content as the prompt + prompt = convert_content_list_to_str(messages[-1]) + + # Create the payload - this is what goes in the body (raw JSON) + payload: dict = {"prompt": prompt} + + # Get or generate session ID - this goes in the header + runtime_session_id = self._get_runtime_session_id(optional_params) + headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = runtime_session_id + + # Get user ID if provided - this goes in the header + runtime_user_id = self._get_runtime_user_id(optional_params) + if runtime_user_id: + headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id + + # The request data is the payload dict (will be JSON encoded by the HTTP handler) + # Qualifier will be handled as a query parameter in get_complete_url + + verbose_logger.debug(f"PAYLOAD: {payload}") + return payload + + def _extract_sse_json(self, line: str) -> Optional[Dict]: + """Extract and parse JSON from an SSE data line.""" + if not line.startswith("data:"): + return None + + json_str = line[5:].strip() + if not json_str: + return None + + try: + data = json.loads(json_str) + # Skip non-dict data (some lines contain JSON strings) + return data if isinstance(data, dict) else None + except json.JSONDecodeError: + verbose_logger.debug(f"Skipping non-JSON line: {line[:100]}") + return None + + def _extract_usage_from_event(self, event_data: Dict) -> Optional[AgentCoreUsage]: + """Extract usage information from event metadata.""" + event_payload = event_data.get("event") + if not event_payload: + return None + + metadata = event_payload.get("metadata") + if metadata and "usage" in metadata: + return metadata["usage"] # type: ignore + + return None + + def _extract_content_delta(self, event_data: Dict) -> Optional[str]: + """Extract text content from contentBlockDelta event.""" + event_payload = event_data.get("event") + if not event_payload: + return None + + content_block_delta = event_payload.get("contentBlockDelta") + if not content_block_delta: + return None + + delta = content_block_delta.get("delta", {}) + return delta.get("text") + + def _extract_content_from_message(self, message: AgentCoreMessage) -> str: + """ + Extract text content from message content blocks. + This works for both SSE messages and JSON responses. + """ + content_list = message.get("content", []) + if not isinstance(content_list, list): + return "" + + return "".join( + block["text"] + for block in content_list + if isinstance(block, dict) and "text" in block + ) + + def _calculate_usage( + self, model: str, messages: List[AllMessageValues], content: str + ) -> Optional[Usage]: + """ + Calculate token usage using LiteLLM's token counter. + + Args: + model: The model name + messages: Input messages + content: Response content + + Returns: + Usage object with calculated tokens, or None if calculation fails + """ + try: + from litellm.utils import token_counter + + prompt_tokens = token_counter(model=model, messages=messages) + completion_tokens = token_counter( + model=model, text=content, count_response_tokens=True + ) + total_tokens = prompt_tokens + completion_tokens + + verbose_logger.debug( + f"Calculated usage - prompt: {prompt_tokens}, completion: {completion_tokens}, total: {total_tokens}" + ) + + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + except Exception as e: + verbose_logger.warning(f"Failed to calculate token usage: {str(e)}") + return None + + def _parse_json_response(self, response_json: dict) -> AgentCoreParsedResponse: + """ + Parse direct JSON response (non-streaming). + + Supports multiple agent response schemas: + 1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore + 2. {"response": [{"text": "..."}]} - Strands agent format + 3. {"result": "plain text"} or {"response": "plain text"} - simple string + 4. Fallback: raw JSON as content string + """ + # Guard: if json.loads() returned a non-dict (e.g. array or primitive), + # skip strategy matching and fall back to raw JSON string + if not isinstance(response_json, dict): + verbose_logger.warning( + "AgentCore: JSON response is not a dict. " + "Returning raw JSON as content." + ) + return AgentCoreParsedResponse( + content=json.dumps(response_json), + usage=None, + final_message=None, + ) + + # Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format + if "result" in response_json and isinstance(response_json["result"], dict): + result = response_json["result"] + content = self._extract_content_from_message(result) # type: ignore + return AgentCoreParsedResponse( + content=content, + usage=None, + final_message=result, # type: ignore + ) + + # Strategy 2: {"response": [{"text": "..."}]} - Strands agent content blocks + if "response" in response_json and isinstance(response_json["response"], list): + content = self._extract_content_from_message( + {"content": response_json["response"]} # type: ignore + ) + return AgentCoreParsedResponse( + content=content, + usage=None, + final_message=None, + ) + + # Strategy 3: string values - {"result": "text"} or {"response": "text"} + for key in ("result", "response"): + val = response_json.get(key) + if isinstance(val, str): + return AgentCoreParsedResponse( + content=val, + usage=None, + final_message=None, + ) + + # Strategy 4: fallback - return raw JSON as content + verbose_logger.warning( + f"AgentCore: Could not extract content from JSON response keys " + f"{list(response_json.keys())}. Returning raw JSON as content." + ) + return AgentCoreParsedResponse( + content=json.dumps(response_json), + usage=None, + final_message=None, + ) + + def _get_parsed_response( + self, raw_response: httpx.Response + ) -> AgentCoreParsedResponse: + """ + Parse AgentCore response based on content type. + + Args: + raw_response: Raw HTTP response from AgentCore + + Returns: + AgentCoreParsedResponse: Parsed response data + """ + content_type = raw_response.headers.get("content-type", "").lower() + verbose_logger.debug(f"AgentCore response Content-Type: {content_type}") + + # Parse response based on content type + if "application/json" in content_type: + # Direct JSON response + verbose_logger.debug("Parsing JSON response") + response_json = raw_response.json() + verbose_logger.debug(f"Response JSON: {response_json}") + return self._parse_json_response(response_json) + else: + # SSE stream response (text/event-stream or default) + verbose_logger.debug("Parsing SSE stream response") + response_text = raw_response.text + verbose_logger.debug( + f"AgentCore response (first 500 chars): {response_text[:500]}" + ) + return self._parse_sse_stream(response_text) + + def _parse_sse_stream(self, response_text: str) -> AgentCoreParsedResponse: + """ + Parse Server-Sent Events (SSE) stream format. + Each line starts with 'data:' followed by JSON. + + Returns: + AgentCoreParsedResponse: Parsed response with content, usage, and message + """ + final_message: Optional[AgentCoreMessage] = None + usage_data: Optional[AgentCoreUsage] = None + content_blocks: List[str] = [] + + for line in response_text.strip().split("\n"): + line = line.strip() + if not line: + continue + + data = self._extract_sse_json(line) + if not data: + continue + + verbose_logger.debug(f"SSE event keys: {list(data.keys())}") + + # Check for final complete message + if "message" in data and isinstance(data["message"], dict): + final_message = data["message"] # type: ignore + verbose_logger.debug("Found final message") + + # Process event data + if "event" in data and isinstance(data["event"], dict): + event_payload = data["event"] + verbose_logger.debug( + f"Event payload keys: {list(event_payload.keys())}" + ) + + # Extract usage metadata + if usage := self._extract_usage_from_event(data): + usage_data = usage + verbose_logger.debug(f"Found usage data: {usage_data}") + + # Collect content deltas + if text := self._extract_content_delta(data): + content_blocks.append(text) + + # Build final content + content = ( + self._extract_content_from_message(final_message) + if final_message + else "".join(content_blocks) + ) + + verbose_logger.debug(f"Final usage_data: {usage_data}") + + return AgentCoreParsedResponse( + content=content, usage=usage_data, final_message=final_message + ) + + def _stream_agentcore_response_sync( + self, + response: httpx.Response, + model: str, + ): + """ + Internal sync generator that parses SSE and yields ModelResponse chunks. + """ + buffer = "" + for text_chunk in response.iter_text(): + buffer += text_chunk + + # Process complete lines + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + line = line.strip() + + if not line or not line.startswith("data:"): + continue + + json_str = line[5:].strip() + if not json_str: + continue + + try: + data_obj = json.loads(json_str) + if not isinstance(data_obj, dict): + continue + + # Process contentBlockDelta events + if "event" in data_obj and isinstance(data_obj["event"], dict): + event_payload = data_obj["event"] + content_block_delta = event_payload.get("contentBlockDelta") + + if content_block_delta: + delta = content_block_delta.get("delta", {}) + text = delta.get("text", "") + + if text: + chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + chunk.choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content=text, role="assistant"), + ) + ] + yield chunk + + # Process metadata/usage + metadata = event_payload.get("metadata") + if metadata and "usage" in metadata: + chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + usage_data: AgentCoreUsage = metadata["usage"] # type: ignore + setattr( + chunk, + "usage", + Usage( + prompt_tokens=usage_data.get("inputTokens", 0), + completion_tokens=usage_data.get("outputTokens", 0), + total_tokens=usage_data.get("totalTokens", 0), + ), + ) + yield chunk + + # Process final message + if "message" in data_obj and isinstance(data_obj["message"], dict): + chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + yield chunk + + except json.JSONDecodeError: + verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + continue + + def get_sync_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional[Union[HTTPHandler, "AsyncHTTPHandler"]] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "CustomStreamWrapper": + """ + Simplified sync streaming - returns a generator that yields ModelResponse chunks. + """ + from litellm.llms.custom_httpx.http_handler import ( + HTTPHandler, + _get_httpx_client, + ) + + if client is None or not isinstance(client, HTTPHandler): + client = _get_httpx_client(params={}) + + verbose_logger.debug(f"Making sync streaming request to: {api_base}") + + # Make streaming request + response = client.post( + api_base, + headers=headers, + data=signed_json_body if signed_json_body else json.dumps(data), + stream=True, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise BedrockError( + status_code=response.status_code, message=str(response.read()) + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + # Check if response is JSON (agent used sync return) instead of SSE + content_type = response.headers.get("content-type", "").lower() + if "application/json" in content_type: + verbose_logger.debug( + "AgentCore streaming: received JSON response instead of SSE, " + "converting to single-chunk stream" + ) + try: + body = response.read() + response_json = json.loads(body) + except (json.JSONDecodeError, Exception) as e: + raise BedrockError( + status_code=response.status_code, + message=f"AgentCore: Failed to read/parse JSON response body: {e}", + ) + parsed = self._parse_json_response(response_json) + + def _json_as_sync_stream(): + # Content chunk + content_chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + content_chunk.choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content=parsed["content"], role="assistant"), + ) + ] + yield content_chunk + + # Stop sentinel chunk (matches SSE path convention) + stop_chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + stop_chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + yield stop_chunk + + return CustomStreamWrapper( + completion_stream=_json_as_sync_stream(), + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + + # SSE stream (text/event-stream or default) - use existing SSE parser + return CustomStreamWrapper( + completion_stream=self._stream_agentcore_response_sync(response, model), + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + + async def _stream_agentcore_response( + self, + response: httpx.Response, + model: str, + ) -> AsyncGenerator[ModelResponseStream, None]: + """ + Internal async generator that parses SSE and yields ModelResponse chunks. + """ + buffer = "" + async for text_chunk in response.aiter_text(): + buffer += text_chunk + + # Process complete lines + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + line = line.strip() + + if not line or not line.startswith("data:"): + continue + + json_str = line[5:].strip() + if not json_str: + continue + + try: + data_obj = json.loads(json_str) + if not isinstance(data_obj, dict): + continue + + # Process contentBlockDelta events + if "event" in data_obj and isinstance(data_obj["event"], dict): + event_payload = data_obj["event"] + content_block_delta = event_payload.get("contentBlockDelta") + + if content_block_delta: + delta = content_block_delta.get("delta", {}) + text = delta.get("text", "") + + if text: + chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + chunk.choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content=text, role="assistant"), + ) + ] + yield chunk + + # Process metadata/usage + metadata = event_payload.get("metadata") + if metadata and "usage" in metadata: + chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + usage_data: AgentCoreUsage = metadata["usage"] # type: ignore + setattr( + chunk, + "usage", + Usage( + prompt_tokens=usage_data.get("inputTokens", 0), + completion_tokens=usage_data.get("outputTokens", 0), + total_tokens=usage_data.get("totalTokens", 0), + ), + ) + yield chunk + + # Process final message + if "message" in data_obj and isinstance(data_obj["message"], dict): + chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + yield chunk + + except json.JSONDecodeError: + verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + continue + + async def get_async_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional["AsyncHTTPHandler"] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "CustomStreamWrapper": + """ + Simplified async streaming - returns an async generator that yields ModelResponse chunks. + """ + from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, + ) + + if client is None or not isinstance(client, AsyncHTTPHandler): + client = get_async_httpx_client( + llm_provider=cast(Any, "bedrock"), params={} + ) + + verbose_logger.debug(f"Making async streaming request to: {api_base}") + + # Make async streaming request + response = await client.post( + api_base, + headers=headers, + data=signed_json_body if signed_json_body else json.dumps(data), + stream=True, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise BedrockError( + status_code=response.status_code, message=str(await response.aread()) + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + # Check if response is JSON (agent used sync return) instead of SSE + content_type = response.headers.get("content-type", "").lower() + if "application/json" in content_type: + verbose_logger.debug( + "AgentCore streaming: received JSON response instead of SSE, " + "converting to single-chunk stream" + ) + try: + body = await response.aread() + response_json = json.loads(body) + except (json.JSONDecodeError, Exception) as e: + raise BedrockError( + status_code=response.status_code, + message=f"AgentCore: Failed to read/parse JSON response body: {e}", + ) + parsed = self._parse_json_response(response_json) + + async def _json_as_async_stream() -> AsyncGenerator[ + ModelResponseStream, None + ]: + # Content chunk + content_chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + content_chunk.choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content=parsed["content"], role="assistant"), + ) + ] + yield content_chunk + + # Stop sentinel chunk (matches SSE path convention) + stop_chunk = ModelResponseStream( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=model, + object="chat.completion.chunk", + ) + stop_chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + yield stop_chunk + + return CustomStreamWrapper( + completion_stream=_json_as_async_stream(), + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + + # SSE stream (text/event-stream or default) - use existing SSE parser + return CustomStreamWrapper( + completion_stream=self._stream_agentcore_response(response, model), + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + + @property + def has_custom_stream_wrapper(self) -> bool: + """Indicates that this config has custom streaming support.""" + return True + + @property + def supports_stream_param_in_request_body(self) -> bool: + """ + AgentCore does not allow passing `stream` in the request body. + Streaming is automatic based on the response format. + """ + return False + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform the AgentCore response to LiteLLM ModelResponse format. + AgentCore can return either JSON or SSE (Server-Sent Events) stream responses. + + Note: For streaming responses, use get_streaming_response() instead. + """ + try: + # Parse the response based on content type (JSON or SSE) + parsed_data = self._get_parsed_response(raw_response) + + content = parsed_data["content"] + usage_data = parsed_data["usage"] + + verbose_logger.debug(f"Parsed content length: {len(content)}") + verbose_logger.debug(f"Usage data: {usage_data}") + + # Create the message + message = Message(content=content, role="assistant") + + # Create choices + choice = Choices(finish_reason="stop", index=0, message=message) + + # Update model response + model_response.choices = [choice] + model_response.model = model + + # Add usage information if available + # Note: AgentCore JSON responses don't include usage data + # SSE responses may include usage in metadata events + if usage_data: + usage = Usage( + prompt_tokens=usage_data.get("inputTokens", 0), + completion_tokens=usage_data.get("outputTokens", 0), + total_tokens=usage_data.get("totalTokens", 0), + ) + setattr(model_response, "usage", usage) + else: + # Calculate token usage using LiteLLM's token counter + verbose_logger.debug( + "No usage data from AgentCore - calculating tokens" + ) + calculated_usage = self._calculate_usage(model, messages, content) + if calculated_usage: + setattr(model_response, "usage", calculated_usage) + + return model_response + + except Exception as e: + verbose_logger.error( + f"Error processing Bedrock AgentCore response: {str(e)}" + ) + raise BedrockError( + message=f"Error processing response: {str(e)}", + status_code=raw_response.status_code, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return BedrockError(status_code=status_code, message=error_message) + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + # AgentCore supports true streaming - don't buffer + return False diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..ef46ae5c18978b12d495354912881a64d8ce2097 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_handler.py @@ -0,0 +1,512 @@ +import json +from typing import Any, Optional, Union + +import httpx + +import litellm +from litellm.anthropic_beta_headers_manager import ( + update_headers_with_filtered_beta, +) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.types.utils import ModelResponse +from litellm.utils import CustomStreamWrapper + +from ..base_aws_llm import BaseAWSLLM, Credentials +from ..common_utils import BedrockError, _get_all_bedrock_regions +from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call + + +def make_sync_call( + client: Optional[HTTPHandler], + api_base: str, + headers: dict, + data: str, + model: str, + messages: list, + logging_obj: LiteLLMLoggingObject, + json_mode: Optional[bool] = False, + fake_stream: bool = False, + stream_chunk_size: int = 1024, +): + if client is None: + client = _get_httpx_client() # Create a new client if none provided + + response = client.post( + api_base, + headers=headers, + data=data, + stream=not fake_stream, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise BedrockError( + status_code=response.status_code, message=str(response.read()) + ) + + if fake_stream: + model_response: ( + ModelResponse + ) = litellm.AmazonConverseConfig()._transform_response( + model=model, + response=response, + model_response=litellm.ModelResponse(), + stream=True, + logging_obj=logging_obj, + optional_params={}, + api_key="", + data=data, + messages=messages, + encoding=litellm.encoding, + ) # type: ignore + completion_stream: Any = MockResponseIterator( + model_response=model_response, json_mode=json_mode + ) + else: + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) + completion_stream = decoder.iter_bytes( + response.iter_bytes(chunk_size=stream_chunk_size) + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return completion_stream + + +class BedrockConverseLLM(BaseAWSLLM): + def __init__(self) -> None: + super().__init__() + + async def async_streaming( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + timeout: Optional[Union[float, httpx.Timeout]], + encoding, + logging_obj, + stream, + optional_params: dict, + litellm_params: dict, + credentials: Credentials, + logger_fn=None, + headers={}, + client: Optional[AsyncHTTPHandler] = None, + fake_stream: bool = False, + json_mode: Optional[bool] = False, + api_key: Optional[str] = None, + stream_chunk_size: int = 1024, + ) -> CustomStreamWrapper: + request_data = await litellm.AmazonConverseConfig()._async_transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + data = json.dumps(request_data) + + prepped = self.get_request_headers( + credentials=credentials, + aws_region_name=litellm_params.get("aws_region_name") or "us-west-2", + extra_headers=headers, + endpoint_url=api_base, + data=data, + headers=headers, + api_key=api_key, + ) + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": dict(prepped.headers), + }, + ) + + completion_stream = await make_call( + client=client, + api_base=api_base, + headers=dict(prepped.headers), + data=data, + model=model, + messages=messages, + logging_obj=logging_obj, + fake_stream=fake_stream, + json_mode=json_mode, + stream_chunk_size=stream_chunk_size, + ) + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + return streaming_response + + async def async_completion( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + timeout: Optional[Union[float, httpx.Timeout]], + encoding, + logging_obj: LiteLLMLoggingObject, + stream, + optional_params: dict, + litellm_params: dict, + credentials: Credentials, + logger_fn=None, + headers: dict = {}, + client: Optional[AsyncHTTPHandler] = None, + api_key: Optional[str] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: + request_data = await litellm.AmazonConverseConfig()._async_transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + data = json.dumps(request_data) + + prepped = self.get_request_headers( + credentials=credentials, + aws_region_name=litellm_params.get("aws_region_name") or "us-west-2", + extra_headers=headers, + endpoint_url=api_base, + data=data, + headers=headers, + api_key=api_key, + ) + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": prepped.headers, + }, + ) + + headers = dict(prepped.headers) + if client is None or not isinstance(client, AsyncHTTPHandler): + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + client = get_async_httpx_client( + params=_params, llm_provider=litellm.LlmProviders.BEDROCK + ) + else: + client = client # type: ignore + + try: + response = await client.post( + url=api_base, + headers=headers, + data=data, + logging_obj=logging_obj, + ) # type: ignore + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=err.response.text) + except httpx.TimeoutException: + raise BedrockError(status_code=408, message="Timeout error occurred.") + + return litellm.AmazonConverseConfig()._transform_response( + model=model, + response=response, + model_response=model_response, + stream=stream if isinstance(stream, bool) else False, + logging_obj=logging_obj, + api_key="", + data=data, + messages=messages, + optional_params=optional_params, + encoding=encoding, + ) + + def completion( # noqa: PLR0915 + self, + model: str, + messages: list, + api_base: Optional[str], + custom_prompt_dict: dict, + model_response: ModelResponse, + encoding, + logging_obj: LiteLLMLoggingObject, + optional_params: dict, + acompletion: bool, + timeout: Optional[Union[float, httpx.Timeout]], + litellm_params: dict, + logger_fn=None, + extra_headers: Optional[dict] = None, + client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None, + api_key: Optional[str] = None, + ): + ## SETUP ## + stream = optional_params.pop("stream", None) + stream_chunk_size = optional_params.pop("stream_chunk_size", 1024) + unencoded_model_id = optional_params.pop("model_id", None) + fake_stream = optional_params.pop("fake_stream", False) + json_mode = optional_params.get("json_mode", False) + if unencoded_model_id is not None: + modelId = self.encode_model_id(model_id=unencoded_model_id) + else: + # Strip nova spec prefixes before encoding model ID for API URL + _model_for_id = model + _stripped = _model_for_id + for rp in ["bedrock/converse/", "bedrock/", "converse/"]: + if _stripped.startswith(rp): + _stripped = _stripped[len(rp) :] + break + # Strip embedded region prefix (e.g. "bedrock/us-east-1/model" -> "model") + # and capture it so it can be used as aws_region_name below. + _region_from_model: Optional[str] = None + _potential_region = _stripped.split("/", 1)[0] + if _potential_region in _get_all_bedrock_regions() and "/" in _stripped: + _region_from_model = _potential_region + _stripped = _stripped.split("/", 1)[1] + _model_for_id = _stripped + for _nova_prefix in ["nova-2/", "nova/"]: + if _stripped.startswith(_nova_prefix): + _model_for_id = _model_for_id.replace(_nova_prefix, "", 1) + break + modelId = self.encode_model_id(model_id=_model_for_id) + # Inject region extracted from model path so _get_aws_region_name picks it up + if ( + _region_from_model is not None + and "aws_region_name" not in optional_params + ): + optional_params["aws_region_name"] = _region_from_model + + fake_stream = litellm.AmazonConverseConfig().should_fake_stream( + fake_stream=fake_stream, + model=model, + stream=stream, + custom_llm_provider="bedrock", + ) + + ### SET REGION NAME ### + aws_region_name = self._get_aws_region_name( + optional_params=optional_params, + model=model, + model_id=unencoded_model_id, + ) + + ## CREDENTIALS ## + # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them + aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) + aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_session_token = optional_params.pop("aws_session_token", None) + aws_role_name = optional_params.pop("aws_role_name", None) + aws_session_name = optional_params.pop("aws_session_name", None) + aws_profile_name = optional_params.pop("aws_profile_name", None) + aws_bedrock_runtime_endpoint = optional_params.pop( + "aws_bedrock_runtime_endpoint", None + ) # https://bedrock-runtime.{region_name}.amazonaws.com + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) + aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None) + aws_external_id = optional_params.pop("aws_external_id", None) + optional_params.pop("aws_region_name", None) + + litellm_params[ + "aws_region_name" + ] = aws_region_name # [DO NOT DELETE] important for async calls + + credentials: Credentials = self.get_credentials( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region_name=aws_region_name, + aws_session_name=aws_session_name, + aws_profile_name=aws_profile_name, + aws_role_name=aws_role_name, + aws_web_identity_token=aws_web_identity_token, + aws_sts_endpoint=aws_sts_endpoint, + aws_external_id=aws_external_id, + ) + + ### SET RUNTIME ENDPOINT ### + endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_region_name=aws_region_name, + ) + if (stream is not None and stream is True) and not fake_stream: + endpoint_url = f"{endpoint_url}/model/{modelId}/converse-stream" + proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse-stream" + else: + endpoint_url = f"{endpoint_url}/model/{modelId}/converse" + proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse" + + ## COMPLETION CALL + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + + # Filter beta headers in HTTP headers before making the request + headers = update_headers_with_filtered_beta( + headers=headers, provider="bedrock_converse" + ) + ### ROUTING (ASYNC, STREAMING, SYNC) + if acompletion: + if isinstance(client, HTTPHandler): + client = None + if stream is True: + return self.async_streaming( + model=model, + messages=messages, + api_base=proxy_endpoint_url, + model_response=model_response, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=True, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + timeout=timeout, + client=client, + json_mode=json_mode, + fake_stream=fake_stream, + credentials=credentials, + api_key=api_key, + stream_chunk_size=stream_chunk_size, + ) # type: ignore + ### ASYNC COMPLETION + return self.async_completion( + model=model, + messages=messages, + api_base=proxy_endpoint_url, + model_response=model_response, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, # type: ignore + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + timeout=timeout, + client=client, + credentials=credentials, + api_key=api_key, + ) # type: ignore + + ## TRANSFORMATION ## + + _data = litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=extra_headers, + ) + data = json.dumps(_data) + + prepped = self.get_request_headers( + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=proxy_endpoint_url, + data=data, + headers=headers, + api_key=api_key, + ) + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": proxy_endpoint_url, + "headers": prepped.headers, + }, + ) + if client is None or isinstance(client, AsyncHTTPHandler): + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + client = _get_httpx_client(_params) # type: ignore + else: + client = client + + if stream is not None and stream is True: + completion_stream = make_sync_call( + client=( + client + if client is not None and isinstance(client, HTTPHandler) + else None + ), + api_base=proxy_endpoint_url, + headers=prepped.headers, # type: ignore + data=data, + model=model, + messages=messages, + logging_obj=logging_obj, + json_mode=json_mode, + fake_stream=fake_stream, + stream_chunk_size=stream_chunk_size, + ) + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + + return streaming_response + + ### COMPLETION + + try: + response = client.post( + url=proxy_endpoint_url, + headers=prepped.headers, + data=data, + logging_obj=logging_obj, + ) # type: ignore + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=err.response.text) + except httpx.TimeoutException: + raise BedrockError(status_code=408, message="Timeout error occurred.") + + return litellm.AmazonConverseConfig()._transform_response( + model=model, + response=response, + model_response=model_response, + stream=stream if isinstance(stream, bool) else False, + logging_obj=logging_obj, + api_key="", + data=data, + messages=messages, + optional_params=optional_params, + encoding=encoding, + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/handler.py new file mode 100644 index 0000000000000000000000000000000000000000..c26886b713f3611777218c7ed3d97d181ccd1b01 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/handler.py @@ -0,0 +1,5 @@ +""" +Uses base_llm_http_handler to call the 'converse like' endpoint. + +Relevant issue: https://github.com/BerriAI/litellm/issues/8085 +""" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..783320224291e915d75533c18389f60ff5a10db2 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_like/transformation.py @@ -0,0 +1,3 @@ +""" +Uses `converse_transformation.py` to transform the messages to the format required by Bedrock Converse. +""" diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..dd8b1b0a69f4bbbed059cd3248e628bb5a86feed --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/converse_transformation.py @@ -0,0 +1,2126 @@ +""" +Translating between OpenAI's `/chat/completion` format and Amazon's `/converse` format +""" + +import copy +import json +import time +import types +from typing import List, Literal, Optional, Tuple, Union, cast, overload + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.constants import ( + BEDROCK_MIN_THINKING_BUDGET_TOKENS, + RESPONSE_FORMAT_TOOL_NAME, +) +from litellm.litellm_core_utils.core_helpers import ( + filter_exceptions_from_params, + filter_internal_params, + map_finish_reason, + safe_deep_copy, +) +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning, +) +from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + _bedrock_converse_messages_pt, + _bedrock_tools_pt, +) +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.types.llms.bedrock import * +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionRedactedThinkingBlock, + ChatCompletionResponseMessage, + ChatCompletionSystemMessage, + ChatCompletionThinkingBlock, + ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, + ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, + ChatCompletionUserMessage, + OpenAIChatCompletionToolParam, + OpenAIMessageContentListBlock, +) +from litellm.types.utils import ( + ChatCompletionMessageToolCall, + CompletionTokensDetailsWrapper, + Function, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) +from litellm.utils import ( + add_dummy_tool, + any_assistant_message_has_thinking_blocks, + has_tool_call_blocks, + last_assistant_with_tool_calls_has_no_thinking_blocks, + supports_reasoning, + token_counter, +) + +from ..common_utils import ( + BedrockError, + BedrockModelInfo, + get_anthropic_beta_from_headers, + get_bedrock_tool_name, + is_claude_4_5_on_bedrock, +) + +# Computer use tool prefixes supported by Bedrock +BEDROCK_COMPUTER_USE_TOOLS = [ + "computer_use_preview", + "computer_", + "bash_", + "text_editor_", +] + +# Beta header patterns that are not supported by Bedrock Converse API +# These will be filtered out to prevent errors +UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS = [ + "advanced-tool-use", # Bedrock Converse doesn't support advanced-tool-use beta headers + "prompt-caching", # Prompt caching not supported in Converse API + "compact-2026-01-12", # The compact beta feature is not currently supported on the Converse and ConverseStream APIs +] + +# Models that support Bedrock's native structured outputs API (outputConfig.textFormat) +# Uses substring matching against the Bedrock model ID +# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html +BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS = { + # Anthropic Claude 4.5+ + "claude-haiku-4-5", + "claude-sonnet-4-5", + "claude-opus-4-5", + "claude-opus-4-6", + # Qwen3 + "qwen3", + # DeepSeek + "deepseek-v3.1", + # Gemma 3 + "gemma-3", + # MiniMax + "minimax-m2", + # Mistral (magistral-small excluded: broken constrained decoding on Bedrock) + "ministral", + "mistral-large-3", + "voxtral", + # Moonshot + "kimi-k2", + # NVIDIA + "nemotron-nano", + # OpenAI (gpt-oss excluded: broken constrained decoding, works via tool-call fallback) +} + + +class AmazonConverseConfig(BaseConfig): + """ + Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html + #2 - https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html#conversation-inference-supported-models-features + """ + + maxTokens: Optional[int] + stopSequences: Optional[List[str]] + temperature: Optional[int] + topP: Optional[int] + topK: Optional[int] + + def __init__( + self, + maxTokens: Optional[int] = None, + stopSequences: Optional[List[str]] = None, + temperature: Optional[int] = None, + topP: Optional[int] = None, + topK: Optional[int] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock_converse" + + @classmethod + def get_config_blocks(cls) -> dict: + return { + "guardrailConfig": GuardrailConfigBlock, + "performanceConfig": PerformanceConfigBlock, + "serviceTier": ServiceTierBlock, + } + + @staticmethod + def _convert_consecutive_user_messages_to_guarded_text( + messages: List[AllMessageValues], optional_params: dict + ) -> List[AllMessageValues]: + """ + Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present + and no guarded_text is already present in those messages. + """ + # Check if guardrailConfig is present + if "guardrailConfig" not in optional_params: + return messages + + # Find all consecutive user messages at the end + consecutive_user_message_indices = [] + for i in range(len(messages) - 1, -1, -1): + if messages[i].get("role") == "user": + consecutive_user_message_indices.append(i) + else: + break + + if not consecutive_user_message_indices: + return messages + + # Process each consecutive user message + messages_copy = copy.deepcopy(messages) + for user_message_index in consecutive_user_message_indices: + user_message = messages_copy[user_message_index] + content = user_message.get("content", []) + + if isinstance(content, list): + has_guarded_text = any( + isinstance(item, dict) and item.get("type") == "guarded_text" + for item in content + ) + if has_guarded_text: + continue # Skip this message if it already has guarded_text + + # Convert text elements to guarded_text + new_content = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + new_item = {"type": "guarded_text", "text": item["text"]} # type: ignore + new_content.append(new_item) + else: + new_content.append(item) + + messages_copy[user_message_index]["content"] = new_content # type: ignore + elif isinstance(content, str): + # If content is a string, convert it to guarded_text + messages_copy[user_message_index]["content"] = [ # type: ignore + {"type": "guarded_text", "text": content} # type: ignore + ] + + return messages_copy + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def _validate_request_metadata(self, metadata: dict) -> None: + """ + Validate requestMetadata according to AWS Bedrock Converse API constraints. + + Constraints: + - Maximum of 16 items + - Keys: 1-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{1,256} + - Values: 0-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{0,256} + """ + import re + + if not isinstance(metadata, dict): + raise litellm.exceptions.BadRequestError( + message="requestMetadata must be a dictionary", + model="bedrock", + llm_provider="bedrock", + ) + + if len(metadata) > 16: + raise litellm.exceptions.BadRequestError( + message="requestMetadata can contain a maximum of 16 items", + model="bedrock", + llm_provider="bedrock", + ) + + key_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$") + value_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$") + + for key, value in metadata.items(): + if not isinstance(key, str): + raise litellm.exceptions.BadRequestError( + message="requestMetadata keys must be strings", + model="bedrock", + llm_provider="bedrock", + ) + + if not isinstance(value, str): + raise litellm.exceptions.BadRequestError( + message="requestMetadata values must be strings", + model="bedrock", + llm_provider="bedrock", + ) + + if len(key) == 0 or len(key) > 256: + raise litellm.exceptions.BadRequestError( + message="requestMetadata key length must be 1-256 characters", + model="bedrock", + llm_provider="bedrock", + ) + + if len(value) > 256: + raise litellm.exceptions.BadRequestError( + message="requestMetadata value length must be 0-256 characters", + model="bedrock", + llm_provider="bedrock", + ) + + if not key_pattern.match(key): + raise litellm.exceptions.BadRequestError( + message=f"requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]", + model="bedrock", + llm_provider="bedrock", + ) + + if not value_pattern.match(value): + raise litellm.exceptions.BadRequestError( + message=f"requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]", + model="bedrock", + llm_provider="bedrock", + ) + + def _is_nova_2_model(self, model: str) -> bool: + """ + Check if the model is a Nova 2 model that supports reasoningConfig. + + Nova 2 models use a different reasoning configuration structure compared to + Anthropic's thinking parameter and GPT-OSS's reasoning_effort parameter. + + Supported models: + - amazon.nova-2-lite-v1:0 + - amazon.nova-2-pro-preview-20251202-v1:0 + - us.amazon.nova-2-lite-v1:0 + - eu.amazon.nova-2-lite-v1:0 + - apac.amazon.nova-2-lite-v1:0 + - (and other regional variants) + + Args: + model: The model identifier + + Returns: + True if the model is a Nova 2 model, False otherwise + + Examples: + >>> config = AmazonConverseConfig() + >>> config._is_nova_2_model("amazon.nova-2-lite-v1:0") + True + >>> config._is_nova_2_model("us.amazon.nova-2-lite-v1:0") + True + >>> config._is_nova_2_model("us.amazon.nova-2-pro-preview-20251202-v1:0") + True + >>> config._is_nova_2_model("amazon.nova-pro-1-5-v1:0") + False + >>> config._is_nova_2_model("amazon.nova-pro-v1:0") + False + """ + # Remove provider routing prefix if present (bedrock/converse/, bedrock/, converse/) + model_without_region = model + for routing_prefix in ["bedrock/converse/", "bedrock/", "converse/"]: + if model_without_region.startswith(routing_prefix): + model_without_region = model_without_region[len(routing_prefix) :] + break + + # Remove regional prefix if present (us., eu., apac.) + for prefix in ["us.", "eu.", "apac."]: + if model_without_region.startswith(prefix): + model_without_region = model_without_region[len(prefix) :] + break + + # Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.) + # Also check for nova-2/ spec prefix for imported models + return model_without_region.startswith( + "amazon.nova-2-" + ) or model_without_region.startswith("nova-2/") + + def _map_web_search_options( + self, web_search_options: dict, model: str + ) -> Optional[BedrockToolBlock]: + """ + Map web_search_options to Nova grounding systemTool. + + Nova grounding (web search) is only supported on Amazon Nova models. + Returns None for non-Nova models. + + Args: + web_search_options: The web_search_options dict from the request + model: The model identifier string + + Returns: + BedrockToolBlock with systemTool for Nova models, None otherwise + + Reference: https://docs.aws.amazon.com/nova/latest/userguide/grounding.html + """ + # Only Nova models support nova_grounding + # Model strings can be like: "amazon.nova-pro-v1:0", "us.amazon.nova-pro-v1:0", etc. + if "nova" not in model.lower(): + verbose_logger.debug( + f"web_search_options passed but model {model} is not a Nova model. " + "Nova grounding is only supported on Amazon Nova models." + ) + return None + + # Nova doesn't support search_context_size or user_location params + # (unlike Anthropic), so we just enable grounding with no options + return BedrockToolBlock(systemTool={"name": "nova_grounding"}) + + def _transform_reasoning_effort_to_reasoning_config( + self, reasoning_effort: str + ) -> dict: + """ + Transform reasoning_effort parameter to Nova 2 reasoningConfig structure. + + Nova 2 models use a reasoningConfig structure in additionalModelRequestFields + that differs from both Anthropic's thinking parameter and GPT-OSS's reasoning_effort. + + Args: + reasoning_effort: The reasoning effort level, must be "low" or "high" + + Returns: + dict: A dictionary containing the reasoningConfig structure: + { + "reasoningConfig": { + "type": "enabled", + "maxReasoningEffort": "low" | "medium" |"high" + } + } + + Raises: + BadRequestError: If reasoning_effort is not "low", "medium" or "high" + + Examples: + >>> config = AmazonConverseConfig() + >>> config._transform_reasoning_effort_to_reasoning_config("high") + {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}} + >>> config._transform_reasoning_effort_to_reasoning_config("low") + {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'low'}} + """ + valid_values = ["low", "medium", "high"] + if reasoning_effort not in valid_values: + raise litellm.exceptions.BadRequestError( + message=f"Invalid reasoning_effort value '{reasoning_effort}' for Nova 2 models. " + f"Supported values: {valid_values}", + model="amazon.nova-2-lite-v1:0", + llm_provider="bedrock_converse", + ) + + return { + "reasoningConfig": { + "type": "enabled", + "maxReasoningEffort": reasoning_effort, + } + } + + def _handle_reasoning_effort_parameter( + self, model: str, reasoning_effort: str, optional_params: dict + ) -> None: + """ + Handle the reasoning_effort parameter based on the model type. + + Different model families handle reasoning effort differently: + - GPT-OSS models: Keep reasoning_effort as-is (passed to additionalModelRequestFields) + - Nova 2 models: Transform to reasoningConfig structure + - Other models (Anthropic, etc.): Convert to thinking parameter + + Args: + model: The model identifier + reasoning_effort: The reasoning effort value + optional_params: Dictionary of optional parameters to update in-place + + Examples: + >>> config = AmazonConverseConfig() + >>> params = {} + >>> config._handle_reasoning_effort_parameter("gpt-oss-model", "high", params) + >>> params + {'reasoning_effort': 'high'} + + >>> params = {} + >>> config._handle_reasoning_effort_parameter("amazon.nova-2-lite-v1:0", "high", params) + >>> params + {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}} + + >>> params = {} + >>> config._handle_reasoning_effort_parameter("anthropic.claude-3", "high", params) + >>> params + {'thinking': {'type': 'enabled', 'budget_tokens': 10000}} + """ + if "gpt-oss" in model: + # GPT-OSS models: keep reasoning_effort as-is + # It will be passed through to additionalModelRequestFields + optional_params["reasoning_effort"] = reasoning_effort + elif self._is_nova_2_model(model): + # Nova 2 models: transform to reasoningConfig + reasoning_config = self._transform_reasoning_effort_to_reasoning_config( + reasoning_effort + ) + optional_params.update(reasoning_config) + else: + # Anthropic and other models: convert to thinking parameter + optional_params["thinking"] = AnthropicConfig._map_reasoning_effort( + reasoning_effort=reasoning_effort, model=model + ) + + @staticmethod + def _clamp_thinking_budget_tokens(optional_params: dict) -> None: + """ + Clamp thinking.budget_tokens to the Bedrock minimum (1024). + + Bedrock returns a 400 error if budget_tokens < 1024. + """ + thinking = optional_params.get("thinking") + if isinstance(thinking, dict): + budget = thinking.get("budget_tokens") + if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS: + verbose_logger.debug( + "Bedrock requires thinking.budget_tokens >= %d, got %d. " + "Clamping to minimum.", + BEDROCK_MIN_THINKING_BUDGET_TOKENS, + budget, + ) + thinking["budget_tokens"] = BEDROCK_MIN_THINKING_BUDGET_TOKENS + + def get_supported_openai_params(self, model: str) -> List[str]: + from litellm.utils import supports_function_calling + + supported_params = [ + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "stop", + "temperature", + "top_p", + "extra_headers", + "response_format", + "requestMetadata", + "service_tier", + "parallel_tool_calls", + ] + + if ( + "arn" in model + ): # we can't infer the model from the arn, so just add all params + supported_params.append("tools") + supported_params.append("tool_choice") + supported_params.append("thinking") + supported_params.append("reasoning_effort") + # For nova imported models, also add web_search_options + if "nova" in model.lower(): + supported_params.append("web_search_options") + return supported_params + + ## Filter out 'cross-region' from model name + base_model = BedrockModelInfo.get_base_model(model) + + if ( + base_model.startswith("anthropic") + or base_model.startswith("mistral") + or base_model.startswith("cohere") + or base_model.startswith("meta.llama3-1") + or base_model.startswith("meta.llama3-2") + or base_model.startswith("meta.llama3-3") + or base_model.startswith("meta.llama4") + or base_model.startswith("amazon.nova") + or supports_function_calling( + model=model, custom_llm_provider=self.custom_llm_provider + ) + ): + supported_params.append("tools") + + # Nova models support web_search_options (mapped to nova_grounding systemTool) + if base_model.startswith("amazon.nova"): + supported_params.append("web_search_options") + + if litellm.utils.supports_tool_choice( + model=model, custom_llm_provider=self.custom_llm_provider + ) or litellm.utils.supports_tool_choice( + model=base_model, custom_llm_provider=self.custom_llm_provider + ): + # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html + supported_params.append("tool_choice") + + if "gpt-oss" in model: + supported_params.append("reasoning_effort") + elif self._is_nova_2_model(model): + # Nova 2 models support reasoning_effort (transformed to reasoningConfig) + # These models use a different reasoning structure than Anthropic's thinking parameter + supported_params.append("reasoning_effort") + elif ( + "claude-3-7" in model + or "claude-sonnet-4" in model + or "claude-opus-4" in model + or "deepseek.r1" in model + or supports_reasoning( + model=model, + custom_llm_provider=self.custom_llm_provider, + ) + or supports_reasoning( + model=base_model, custom_llm_provider=self.custom_llm_provider + ) + ): + supported_params.append("thinking") + supported_params.append("reasoning_effort") + return supported_params + + def map_tool_choice_values( + self, model: str, tool_choice: Union[str, dict], drop_params: bool + ) -> Optional[ToolChoiceValuesBlock]: + if tool_choice == "none": + if litellm.drop_params is True or drop_params is True: + return None + else: + raise litellm.utils.UnsupportedParamsError( + message="Bedrock doesn't support tool_choice={}. To drop it from the call, set `litellm.drop_params = True.".format( + tool_choice + ), + status_code=400, + ) + elif tool_choice == "required": + return ToolChoiceValuesBlock(any={}) + elif tool_choice == "auto": + return ToolChoiceValuesBlock(auto={}) + elif isinstance(tool_choice, dict): + # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html + specific_tool = SpecificToolChoiceBlock( + name=tool_choice.get("function", {}).get("name", "") + ) + return ToolChoiceValuesBlock(tool=specific_tool) + else: + raise litellm.utils.UnsupportedParamsError( + message="Bedrock doesn't support tool_choice={}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.".format( + tool_choice + ), + status_code=400, + ) + + def get_supported_image_types(self) -> List[str]: + return ["png", "jpeg", "gif", "webp"] + + def get_supported_document_types(self) -> List[str]: + return ["pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"] + + def get_supported_video_types(self) -> List[str]: + return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"] + + def get_all_supported_content_types(self) -> List[str]: + return ( + self.get_supported_image_types() + + self.get_supported_document_types() + + self.get_supported_video_types() + ) + + def is_computer_use_tool_used( + self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str + ) -> bool: + """Check if computer use tools are being used in the request.""" + if tools is None: + return False + + for tool in tools: + if "type" in tool: + tool_type = tool["type"] + for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS: + if tool_type.startswith(computer_use_prefix): + return True + return False + + def _transform_computer_use_tools( + self, computer_use_tools: List[OpenAIChatCompletionToolParam] + ) -> List[dict]: + """Transform computer use tools to Bedrock format.""" + transformed_tools: List[dict] = [] + + for tool in computer_use_tools: + tool_type = tool.get("type", "") + + # Check if this is a computer use tool with the startswith method + is_computer_use_tool = False + for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS: + if tool_type.startswith(computer_use_prefix): + is_computer_use_tool = True + break + + transformed_tool: dict = {} + if is_computer_use_tool: + if tool_type.startswith("computer_") and "function" in tool: + # Computer use tool with function format + func = tool["function"] + transformed_tool = { + "type": tool_type, + "name": func.get("name", "computer"), + **func.get("parameters", {}), + } + else: + # Direct tools - just need to ensure name is present + transformed_tool = dict(tool) + if "name" not in transformed_tool: + if tool_type.startswith("bash_"): + transformed_tool["name"] = "bash" + elif tool_type.startswith("text_editor_"): + transformed_tool["name"] = "str_replace_editor" + else: + # Pass through other tools as-is + transformed_tool = dict(tool) + + transformed_tools.append(transformed_tool) + + return transformed_tools + + def _separate_computer_use_tools( + self, tools: List[OpenAIChatCompletionToolParam], model: str + ) -> Tuple[ + List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam] + ]: + """ + Separate computer use tools from regular function tools. + + Args: + tools: List of tools to separate + model: The model name to check if it supports computer use + + Returns: + Tuple of (computer_use_tools, regular_tools) + """ + computer_use_tools = [] + regular_tools = [] + + for tool in tools: + if "type" in tool: + tool_type = tool["type"] + is_computer_use_tool = False + for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS: + if tool_type.startswith(computer_use_prefix): + is_computer_use_tool = True + break + if is_computer_use_tool: + computer_use_tools.append(tool) + else: + regular_tools.append(tool) + else: + regular_tools.append(tool) + + return computer_use_tools, regular_tools + + def _create_json_tool_call_for_response_format( + self, + json_schema: Optional[dict] = None, + description: Optional[str] = None, + ) -> ChatCompletionToolParam: + """ + Handles creating a tool call for getting responses in JSON format. + + Args: + json_schema (Optional[dict]): The JSON schema the response should be in + + Returns: + AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format + """ + + if json_schema is None: + # Anthropic raises a 400 BadRequest error if properties is passed as None + # see usage with additionalProperties (Example 5) https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb + _input_schema = { + "type": "object", + "additionalProperties": True, + "properties": {}, + } + else: + # Use the schema as-is for Bedrock + # Bedrock requires the tool schema to be of type "object" and doesn't need unwrapping + _input_schema = json_schema + + tool_param_function_chunk = ChatCompletionToolParamFunctionChunk( + name=RESPONSE_FORMAT_TOOL_NAME, parameters=_input_schema + ) + if description: + tool_param_function_chunk["description"] = description + + _tool = ChatCompletionToolParam( + type="function", + function=tool_param_function_chunk, + ) + return _tool + + @staticmethod + def _supports_native_structured_outputs(model: str) -> bool: + """Check if the Bedrock model supports native structured outputs (outputConfig.textFormat).""" + return any( + substring in model for substring in BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS + ) + + @staticmethod + def _add_additional_properties_to_schema(schema: dict) -> dict: + """ + Recursively ensure all object types in a JSON schema have + ``"additionalProperties": false``. + + Bedrock's native structured-outputs API requires this field to be + explicitly set on every object node, otherwise it returns a + validation error. + """ + if not isinstance(schema, dict): + return schema + + result = dict(schema) + + if result.get("type") == "object" and "additionalProperties" not in result: + result["additionalProperties"] = False + + # Recurse into nested schemas + if "properties" in result and isinstance(result["properties"], dict): + result["properties"] = { + k: AmazonConverseConfig._add_additional_properties_to_schema(v) + for k, v in result["properties"].items() + } + if "items" in result and isinstance(result["items"], dict): + result["items"] = AmazonConverseConfig._add_additional_properties_to_schema( + result["items"] + ) + for defs_key in ("$defs", "definitions"): + if defs_key in result and isinstance(result[defs_key], dict): + result[defs_key] = { + k: AmazonConverseConfig._add_additional_properties_to_schema(v) + for k, v in result[defs_key].items() + } + for key in ("anyOf", "allOf", "oneOf"): + if key in result and isinstance(result[key], list): + result[key] = [ + AmazonConverseConfig._add_additional_properties_to_schema(item) + for item in result[key] + ] + + return result + + @staticmethod + def _create_output_config_for_response_format( + json_schema: Optional[dict] = None, + name: Optional[str] = None, + description: Optional[str] = None, + ) -> "OutputConfigBlock": + """ + Build an outputConfig block for Bedrock's native structured outputs API. + + The Converse API expects: + { + "outputConfig": { + "textFormat": { + "type": "json_schema", + "structure": { + "jsonSchema": { + "schema": "", + "name": "optional", + "description": "optional" + } + } + } + } + } + """ + if json_schema is not None: + json_schema = AmazonConverseConfig._add_additional_properties_to_schema( + json_schema + ) + schema_str = json.dumps(json_schema) if json_schema is not None else "{}" + json_schema_def: JsonSchemaDefinition = {"schema": schema_str} + if name is not None: + json_schema_def["name"] = name + if description is not None: + json_schema_def["description"] = description + + return OutputConfigBlock( + textFormat=OutputFormat( + type="json_schema", + structure=OutputFormatStructure(jsonSchema=json_schema_def), + ) + ) + + def _apply_tool_call_transformation( + self, + tools: List[OpenAIChatCompletionToolParam], + model: str, + non_default_params: dict, + optional_params: dict, + ): + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, tools=tools + ) + + if ( + "meta.llama3-3-70b-instruct-v1:0" in model + and non_default_params.get("stream", False) is True + ): + optional_params["fake_stream"] = True + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + is_thinking_enabled = self.is_thinking_enabled(non_default_params) + + for param, value in non_default_params.items(): + if param == "response_format" and isinstance(value, dict): + optional_params = self._translate_response_format_param( + value=value, + model=model, + optional_params=optional_params, + non_default_params=non_default_params, + is_thinking_enabled=is_thinking_enabled, + ) + if param == "max_tokens" or param == "max_completion_tokens": + optional_params["maxTokens"] = value + if param == "stream": + optional_params["stream"] = value + if param == "stop": + if isinstance(value, str): + if len(value) == 0: # converse raises error for empty strings + continue + value = [value] + optional_params["stopSequences"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["topP"] = value + if param == "tools" and isinstance(value, list): + self._apply_tool_call_transformation( + tools=cast(List[OpenAIChatCompletionToolParam], value), + model=model, + non_default_params=non_default_params, + optional_params=optional_params, + ) + if param == "tool_choice": + _tool_choice_value = self.map_tool_choice_values( + model=model, tool_choice=value, drop_params=drop_params # type: ignore + ) + if _tool_choice_value is not None: + optional_params["tool_choice"] = _tool_choice_value + if param == "parallel_tool_calls": + disable_parallel = not value + optional_params["_parallel_tool_use_config"] = { + "tool_choice": {"disable_parallel_tool_use": disable_parallel} + } + if param == "thinking": + optional_params["thinking"] = value + elif param == "reasoning_effort" and isinstance(value, str): + self._handle_reasoning_effort_parameter( + model=model, reasoning_effort=value, optional_params=optional_params + ) + if param == "requestMetadata": + if value is not None and isinstance(value, dict): + self._validate_request_metadata(value) # type: ignore + optional_params["requestMetadata"] = value + if param == "service_tier" and isinstance(value, str): + self._map_service_tier_param(value, optional_params) + + if param == "web_search_options" and isinstance(value, dict): + # Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)` + # because empty dict {} is falsy but is a valid way to enable Nova grounding + grounding_tool = self._map_web_search_options(value, model) + if grounding_tool is not None: + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, tools=[grounding_tool] + ) + + # Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models + # Nova 2 handles token budgeting differently through reasoningConfig + if "gpt-oss" not in model and not self._is_nova_2_model(model): + self.update_optional_params_with_thinking_tokens( + non_default_params=non_default_params, optional_params=optional_params + ) + + final_is_thinking_enabled = self.is_thinking_enabled(optional_params) + if final_is_thinking_enabled and "tool_choice" in optional_params: + tool_choice_block = optional_params["tool_choice"] + if isinstance(tool_choice_block, dict): + if "any" in tool_choice_block or "tool" in tool_choice_block: + verbose_logger.info( + f"{model} does not support forced tool use (tool_choice='required' or specific tool) " + f"when reasoning is enabled. Changing tool_choice to 'auto'." + ) + optional_params["tool_choice"] = ToolChoiceValuesBlock(auto={}) + + return optional_params + + def _map_service_tier_param(self, value: str, optional_params: dict) -> None: + """Map OpenAI service_tier (string) to Bedrock serviceTier (object). + + OpenAI values: "auto", "default", "flex", "priority" + Bedrock values: "default", "flex", "priority" (no "auto") + """ + bedrock_tier = value + if value == "auto": + bedrock_tier = "default" # Bedrock doesn't support "auto" + if bedrock_tier in ("default", "flex", "priority"): + optional_params["serviceTier"] = {"type": bedrock_tier} + + def _translate_response_format_param( + self, + value: dict, + model: str, + optional_params: dict, + non_default_params: dict, + is_thinking_enabled: bool, + ) -> dict: + """ + Handles translation of response_format parameter to Bedrock format. + + Returns `optional_params` with the translated response_format parameter. + """ + ignore_response_format_types = ["text"] + if value["type"] in ignore_response_format_types: # value is a no-op + return optional_params + + json_schema: Optional[dict] = None + name: Optional[str] = None + description: Optional[str] = None + if "response_schema" in value: + json_schema = value["response_schema"] + elif "json_schema" in value: + json_schema = value["json_schema"]["schema"] + name = value["json_schema"].get("name") + description = value["json_schema"].get("description") + + if "type" in value and value["type"] == "text": + return optional_params + + if self._supports_native_structured_outputs(model) and json_schema is not None: + # Use Bedrock's native structured outputs API (outputConfig.textFormat) + # No synthetic tool injection, no fake_stream needed. + # Requires an explicit schema — json_object with no schema falls through + # to the tool-call path below. + output_config = self._create_output_config_for_response_format( + json_schema=json_schema, + name=name, + description=description, + ) + optional_params["outputConfig"] = output_config + else: + # Fallback: translate to a synthetic tool call + # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + _tool = self._create_json_tool_call_for_response_format( + json_schema=json_schema, + description=description, + ) + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, tools=[_tool] + ) + + if ( + litellm.utils.supports_tool_choice( + model=model, custom_llm_provider=self.custom_llm_provider + ) + and not is_thinking_enabled + ): + optional_params["tool_choice"] = ToolChoiceValuesBlock( + tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) + ) + if non_default_params.get("stream", False) is True: + optional_params["fake_stream"] = True + + optional_params["json_mode"] = True + return optional_params + + def update_optional_params_with_thinking_tokens( + self, non_default_params: dict, optional_params: dict + ): + """ + Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget. + + Checks 'non_default_params' for 'thinking' and 'max_tokens' + + if 'thinking' is enabled and 'max_tokens' is not specified, set 'max_tokens' to the thinking token budget + DEFAULT_MAX_TOKENS + + Also clamps thinking.budget_tokens to the Bedrock minimum (1024) to + prevent 400 errors from the Bedrock API. + """ + from litellm.constants import DEFAULT_MAX_TOKENS + + self._clamp_thinking_budget_tokens(optional_params) + + is_thinking_enabled = self.is_thinking_enabled(optional_params) + is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params) + if is_thinking_enabled and not is_max_tokens_in_request: + thinking_token_budget = cast(dict, optional_params["thinking"]).get( + "budget_tokens", None + ) + if thinking_token_budget is not None: + optional_params["maxTokens"] = ( + thinking_token_budget + DEFAULT_MAX_TOKENS + ) + + @overload + def _get_cache_point_block( + self, + message_block: Union[ + OpenAIMessageContentListBlock, + ChatCompletionUserMessage, + ChatCompletionSystemMessage, + ChatCompletionAssistantMessage, + ], + block_type: Literal["system"], + model: Optional[str] = None, + ) -> Optional[SystemContentBlock]: + pass + + @overload + def _get_cache_point_block( + self, + message_block: Union[ + OpenAIMessageContentListBlock, + ChatCompletionUserMessage, + ChatCompletionSystemMessage, + ChatCompletionAssistantMessage, + ], + block_type: Literal["content_block"], + model: Optional[str] = None, + ) -> Optional[ContentBlock]: + pass + + def _get_cache_point_block( + self, + message_block: Union[ + OpenAIMessageContentListBlock, + ChatCompletionUserMessage, + ChatCompletionSystemMessage, + ChatCompletionAssistantMessage, + ], + block_type: Literal["system", "content_block"], + model: Optional[str] = None, + ) -> Optional[Union[SystemContentBlock, ContentBlock]]: + cache_control = message_block.get("cache_control", None) + if cache_control is None: + return None + + cache_point = CachePointBlock(type="default") + if isinstance(cache_control, dict) and "ttl" in cache_control: + ttl = cache_control["ttl"] + if ttl in ["5m", "1h"] and model is not None: + if is_claude_4_5_on_bedrock(model): + cache_point["ttl"] = ttl + + if block_type == "system": + return SystemContentBlock(cachePoint=cache_point) + else: + return ContentBlock(cachePoint=cache_point) + + def _transform_system_message( + self, messages: List[AllMessageValues], model: Optional[str] = None + ) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]: + system_prompt_indices = [] + system_content_blocks: List[SystemContentBlock] = [] + for idx, message in enumerate(messages): + if message["role"] == "system": + system_prompt_indices.append(idx) + if isinstance(message["content"], str) and message["content"]: + system_content_blocks.append( + SystemContentBlock(text=message["content"]) + ) + cache_block = self._get_cache_point_block( + message, block_type="system", model=model + ) + if cache_block: + system_content_blocks.append(cache_block) + elif isinstance(message["content"], list): + for m in message["content"]: + if m.get("type") == "text" and m.get("text"): + system_content_blocks.append( + SystemContentBlock(text=m["text"]) + ) + cache_block = self._get_cache_point_block( + m, block_type="system", model=model + ) + if cache_block: + system_content_blocks.append(cache_block) + if len(system_prompt_indices) > 0: + for idx in reversed(system_prompt_indices): + messages.pop(idx) + return messages, system_content_blocks + + def _transform_inference_params(self, inference_params: dict) -> InferenceConfig: + if "top_k" in inference_params: + inference_params["topK"] = inference_params.pop("top_k") + return InferenceConfig(**inference_params) + + def _handle_top_k_value(self, model: str, inference_params: dict) -> dict: + base_model = BedrockModelInfo.get_base_model(model) + + val_top_k = None + if "topK" in inference_params: + val_top_k = inference_params.pop("topK") + elif "top_k" in inference_params: + val_top_k = inference_params.pop("top_k") + + if val_top_k: + if base_model.startswith("anthropic"): + return {"top_k": val_top_k} + if base_model.startswith("amazon.nova"): + return {"inferenceConfig": {"topK": val_top_k}} + + return {} + + def _prepare_request_params( + self, optional_params: dict, model: str + ) -> Tuple[dict, dict, dict, Optional[OutputConfigBlock]]: + """Prepare and separate request parameters.""" + # Filter out exception objects before deepcopy to prevent deepcopy failures + # Exceptions should not be stored in optional_params (this is a defensive fix) + cleaned_params = filter_exceptions_from_params(optional_params) + inference_params = safe_deep_copy(cleaned_params) + supported_converse_params = list( + AmazonConverseConfig.__annotations__.keys() + ) + ["top_k"] + supported_tool_call_params = ["tools", "tool_choice"] + supported_config_params = list(self.get_config_blocks().keys()) + total_supported_params = ( + supported_converse_params + + supported_tool_call_params + + supported_config_params + ) + inference_params.pop("json_mode", None) # used for handling json_schema + # Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and + # will reject `output_config` if it leaks through pass-through routes. + inference_params.pop("output_config", None) + + # Extract requestMetadata before processing other parameters + request_metadata = inference_params.pop("requestMetadata", None) + if request_metadata is not None: + self._validate_request_metadata(request_metadata) + + output_config: Optional[OutputConfigBlock] = inference_params.pop( + "outputConfig", None + ) + inference_params.pop( + "output_config", None + ) # Bedrock Converse doesn't support it + + # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' + additional_request_params = { + k: v for k, v in inference_params.items() if k not in total_supported_params + } + inference_params = { + k: v for k, v in inference_params.items() if k in total_supported_params + } + + # Handle parallel_tool_calls configuration + parallel_tool_use_config = additional_request_params.pop( + "_parallel_tool_use_config", None + ) + if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): + for key, value in parallel_tool_use_config.items(): + if ( + key in additional_request_params + and isinstance(additional_request_params[key], dict) + and isinstance(value, dict) + ): + additional_request_params[key].update(value) + else: + additional_request_params[key] = value + + additional_request_params.pop("parallel_tool_calls", None) + + # Only set the topK value in for models that support it + additional_request_params.update( + self._handle_top_k_value(model, inference_params) + ) + + # Filter out internal/MCP-related parameters that shouldn't be sent to the API + # These are LiteLLM internal parameters, not API parameters + additional_request_params = filter_internal_params(additional_request_params) + + # Filter out non-serializable objects (exceptions, callables, logging objects, etc.) + # from additional_request_params to prevent JSON serialization errors + # This filters: Exception objects, callable objects (functions), Logging objects, etc. + additional_request_params = filter_exceptions_from_params( + additional_request_params + ) + + return ( + inference_params, + additional_request_params, + request_metadata, + output_config, + ) + + def _process_tools_and_beta( + self, + original_tools: list, + model: str, + headers: Optional[dict], + additional_request_params: dict, + ) -> Tuple[List[ToolBlock], list]: + """Process tools and collect anthropic_beta values.""" + bedrock_tools: List[ToolBlock] = [] + + # Collect anthropic_beta values from user headers + anthropic_beta_list = [] + if headers: + user_betas = get_anthropic_beta_from_headers(headers) + anthropic_beta_list.extend(user_betas) + + # Separate pre-formatted Bedrock tools (e.g. systemTool from web_search_options) + # from OpenAI-format tools that need transformation via _bedrock_tools_pt + filtered_tools = [] + pre_formatted_tools: List[ToolBlock] = [] + if original_tools: + for tool in original_tools: + # Already-formatted Bedrock tools (e.g. systemTool for Nova grounding) + if "systemTool" in tool: + pre_formatted_tools.append(tool) + continue + tool_type = tool.get("type", "") + if tool_type in ( + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", + ): + # Tool search not supported in Converse API - skip it + continue + filtered_tools.append(tool) + + # Only separate tools if computer use tools are actually present + if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): + # Separate computer use tools from regular function tools + computer_use_tools, regular_tools = self._separate_computer_use_tools( + filtered_tools, model + ) + + # Process regular function tools using existing logic + bedrock_tools = _bedrock_tools_pt(regular_tools) + + # Add computer use tools and anthropic_beta if needed (only when computer use tools are present) + if computer_use_tools: + # Determine the correct computer-use beta header based on model + # "computer-use-2025-11-24" for Claude Opus 4.6, Claude Opus 4.5 + # "computer-use-2025-01-24" for Claude Sonnet 4.5, Haiku 4.5, Opus 4.1, Sonnet 4, Opus 4, and Sonnet 3.7 + # "computer-use-2024-10-22" for older models + model_lower = model.lower() + if ( + "opus-4.6" in model_lower + or "opus_4.6" in model_lower + or "opus-4-6" in model_lower + or "opus_4_6" in model_lower + or "sonnet-4.6" in model_lower + or "sonnet_4.6" in model_lower + or "sonnet-4-6" in model_lower + or "sonnet_4_6" in model_lower + ): + computer_use_header = "computer-use-2025-11-24" + elif ( + "opus-4.5" in model_lower + or "opus_4.5" in model_lower + or "opus-4-5" in model_lower + or "opus_4_5" in model_lower + ): + computer_use_header = "computer-use-2025-11-24" + elif any( + pattern in model_lower + for pattern in [ + "sonnet-4.5", + "sonnet_4.5", + "sonnet-4-5", + "sonnet_4_5", + "haiku-4.5", + "haiku_4.5", + "haiku-4-5", + "haiku_4_5", + "opus-4.1", + "opus_4.1", + "opus-4-1", + "opus_4_1", + "sonnet-4", + "sonnet_4", + "opus-4", + "opus_4", + "sonnet-3.7", + "sonnet_3.7", + "sonnet-3-7", + "sonnet_3_7", + ] + ): + computer_use_header = "computer-use-2025-01-24" + else: + computer_use_header = "computer-use-2024-10-22" + + anthropic_beta_list.append(computer_use_header) + # Transform computer use tools to proper Bedrock format + transformed_computer_tools = self._transform_computer_use_tools( + computer_use_tools + ) + additional_request_params["tools"] = transformed_computer_tools + else: + # No computer use tools, process all tools as regular tools + bedrock_tools = _bedrock_tools_pt(filtered_tools) + + # Append pre-formatted tools (systemTool etc.) after transformation + bedrock_tools.extend(pre_formatted_tools) + + # Set anthropic_beta in additional_request_params if we have any beta features + # ONLY apply to Anthropic/Claude models - other models (e.g., Qwen, Llama) don't support this field + base_model = BedrockModelInfo.get_base_model(model) + if anthropic_beta_list and base_model.startswith("anthropic"): + additional_request_params["anthropic_beta"] = anthropic_beta_list + + return bedrock_tools, anthropic_beta_list + + def _transform_request_helper( + self, + model: str, + system_content_blocks: List[SystemContentBlock], + optional_params: dict, + messages: Optional[List[AllMessageValues]] = None, + headers: Optional[dict] = None, + ) -> CommonRequestObject: + ## VALIDATE REQUEST + """ + Bedrock doesn't support tool calling without `tools=` param specified. + """ + if ( + "tools" not in optional_params + and messages is not None + and has_tool_call_blocks(messages) + ): + if litellm.modify_params: + optional_params["tools"] = add_dummy_tool( + custom_llm_provider="bedrock_converse" + ) + else: + raise litellm.UnsupportedParamsError( + message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.", + model="", + llm_provider="bedrock", + ) + + # Drop thinking param if thinking is enabled but thinking_blocks are missing + # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use" + # + # IMPORTANT: Only drop thinking if NO assistant messages have thinking_blocks. + # If any message has thinking_blocks, we must keep thinking enabled, otherwise + # Related issues: https://github.com/BerriAI/litellm/issues/14194 + if ( + optional_params.get("thinking") is not None + and messages is not None + and last_assistant_with_tool_calls_has_no_thinking_blocks(messages) + and not any_assistant_message_has_thinking_blocks(messages) + ): + if litellm.modify_params: + optional_params.pop("thinking", None) + litellm.verbose_logger.warning( + "Dropping 'thinking' param because the last assistant message with tool_calls " + "has no thinking_blocks. The model won't use extended thinking for this turn." + ) + + # Prepare and separate parameters + ( + inference_params, + additional_request_params, + request_metadata, + output_config, + ) = self._prepare_request_params(optional_params, model) + + original_tools = inference_params.pop("tools", []) + + # Process tools and collect beta values + bedrock_tools, anthropic_beta_list = self._process_tools_and_beta( + original_tools, model, headers, additional_request_params + ) + + # Append cachePoint to tools if cache_control_injection_points has tool_config + cache_injection_points = additional_request_params.pop( + "cache_control_injection_points", None + ) + if cache_injection_points and len(bedrock_tools) > 0: + for point in cache_injection_points: + if point.get("location") == "tool_config": + bedrock_tools.append({"cachePoint": {"type": "default"}}) + break + + bedrock_tool_config: Optional[ToolConfigBlock] = None + if len(bedrock_tools) > 0: + tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( + "tool_choice", None + ) + bedrock_tool_config = ToolConfigBlock( + tools=bedrock_tools, + ) + if tool_choice_values is not None: + bedrock_tool_config["toolChoice"] = tool_choice_values + + data: CommonRequestObject = { + "additionalModelRequestFields": additional_request_params, + "system": system_content_blocks, + "inferenceConfig": self._transform_inference_params( + inference_params=inference_params + ), + } + + # Handle all config blocks + for config_name, config_class in self.get_config_blocks().items(): + config_value = inference_params.pop(config_name, None) + if config_value is not None: + data[config_name] = config_class(**config_value) # type: ignore + + # Tool Config + if bedrock_tool_config is not None: + data["toolConfig"] = bedrock_tool_config + + # Request Metadata (top-level field) + if request_metadata is not None: + data["requestMetadata"] = request_metadata + + if output_config is not None: + data["outputConfig"] = output_config + + return data + + async def _async_transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: Optional[dict] = None, + ) -> RequestObject: + messages, system_content_blocks = self._transform_system_message( + messages, model=model + ) + + # Convert last user message to guarded_text if guardrailConfig is present + messages = self._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) + ## TRANSFORMATION ## + + _data: CommonRequestObject = self._transform_request_helper( + model=model, + system_content_blocks=system_content_blocks, + optional_params=optional_params, + messages=messages, + headers=headers, + ) + + bedrock_messages = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model=model, + llm_provider="bedrock_converse", + user_continue_message=litellm_params.pop("user_continue_message", None), + ) + ) + + data: RequestObject = {"messages": bedrock_messages, **_data} + + return data + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + return cast( + dict, + self._transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ), + ) + + def _transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: Optional[dict] = None, + ) -> RequestObject: + messages, system_content_blocks = self._transform_system_message( + messages, model=model + ) + + # Convert last user message to guarded_text if guardrailConfig is present + messages = self._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) + + _data: CommonRequestObject = self._transform_request_helper( + model=model, + system_content_blocks=system_content_blocks, + optional_params=optional_params, + messages=messages, + headers=headers, + ) + + ## TRANSFORMATION ## + bedrock_messages: List[MessageBlock] = _bedrock_converse_messages_pt( + messages=messages, + model=model, + llm_provider="bedrock_converse", + user_continue_message=litellm_params.pop("user_continue_message", None), + ) + + data: RequestObject = {"messages": bedrock_messages, **_data} + + return data + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: Logging, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + return self._transform_response( + model=model, + response=raw_response, + model_response=model_response, + stream=optional_params.get("stream", False), + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + data=request_data, + messages=messages, + encoding=encoding, + ) + + def _transform_reasoning_content( + self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock] + ) -> str: + """ + Extract the reasoning text from the reasoning content blocks + + Ensures deepseek reasoning content compatible output. + """ + reasoning_content_str = "" + for block in reasoning_content_blocks: + if "reasoningText" in block: + reasoning_content_str += block["reasoningText"]["text"] + return reasoning_content_str + + def _transform_thinking_blocks( + self, thinking_blocks: List[BedrockConverseReasoningContentBlock] + ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]: + """Return a consistent format for thinking blocks between Anthropic and Bedrock.""" + thinking_blocks_list: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = [] + for block in thinking_blocks: + if "reasoningText" in block: + _thinking_block = ChatCompletionThinkingBlock(type="thinking") + _text = block["reasoningText"].get("text") + _signature = block["reasoningText"].get("signature") + if _text is not None: + _thinking_block["thinking"] = _text + if _signature is not None: + _thinking_block["signature"] = _signature + thinking_blocks_list.append(_thinking_block) + elif "redactedContent" in block: + _redacted_block = ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", data=block["redactedContent"] + ) + thinking_blocks_list.append(_redacted_block) + return thinking_blocks_list + + def _transform_usage( + self, + usage: ConverseTokenUsageBlock, + reasoning_content: Optional[str] = None, + ) -> Usage: + input_tokens = usage["inputTokens"] + output_tokens = usage["outputTokens"] + total_tokens = usage["totalTokens"] + cache_creation_input_tokens: int = 0 + cache_read_input_tokens: int = 0 + + if "cacheReadInputTokens" in usage: + cache_read_input_tokens = usage["cacheReadInputTokens"] + input_tokens += cache_read_input_tokens + if "cacheWriteInputTokens" in usage: + cache_creation_input_tokens = usage["cacheWriteInputTokens"] + input_tokens += cache_creation_input_tokens + + prompt_tokens_details = PromptTokensDetailsWrapper( + cached_tokens=cache_read_input_tokens + ) + reasoning_tokens = ( + token_counter(text=reasoning_content, count_response_tokens=True) + if reasoning_content + else 0 + ) + completion_tokens_details = CompletionTokensDetailsWrapper( + reasoning_tokens=reasoning_tokens, + text_tokens=( + output_tokens - reasoning_tokens + if reasoning_tokens > 0 + else output_tokens + ), + ) + openai_usage = Usage( + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + cache_creation_input_tokens=cache_creation_input_tokens, + cache_read_input_tokens=cache_read_input_tokens, + completion_tokens_details=completion_tokens_details, + ) + return openai_usage + + def get_tool_call_names( + self, + tools: Optional[ + Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]] + ] = None, + ) -> List[str]: + if tools is None: + return [] + tool_set: set[str] = set() + for tool in tools: + tool_spec = tool.get("toolSpec") + function = tool.get("function") + if tool_spec is not None: + _name = cast(dict, tool_spec).get("name") + if _name is not None and isinstance(_name, str): + tool_set.add(_name) + if function is not None: + _name = cast(dict, function).get("name") + if _name is not None and isinstance(_name, str): + tool_set.add(_name) + return list(tool_set) + + def apply_tool_call_transformation_if_needed( + self, + message: Message, + tools: Optional[List[ToolBlock]] = None, + initial_finish_reason: Optional[str] = None, + ) -> Tuple[Message, Optional[str]]: + """ + Apply tool call transformation to a message. + + LLM providers (e.g. Bedrock, Vertex AI) sometimes return tool call in the response content. + + If the response content is a JSON object, we can parse it and return the tool call in the tool_calls field. + """ + returned_finish_reason = initial_finish_reason + if tools is None: + return message, returned_finish_reason + + if message.content is not None: + try: + tool_call_names = self.get_tool_call_names(tools) + json_content = json.loads(message.content) + if ( + json_content.get("type") == "function" + and json_content.get("name") in tool_call_names + ): + tool_calls = [ + ChatCompletionMessageToolCall(function=Function(**json_content)) + ] + + message.tool_calls = tool_calls + message.content = None + returned_finish_reason = "tool_calls" + except Exception: + pass + + return message, returned_finish_reason + + def _translate_message_content( + self, content_blocks: List[ContentBlock] + ) -> Tuple[ + str, + List[ChatCompletionToolCallChunk], + Optional[List[BedrockConverseReasoningContentBlock]], + Optional[List[CitationsContentBlock]], + ]: + """ + Translate the message content to a string and a list of tool calls, reasoning content blocks, and citations. + + Returns: + content_str: str + tools: List[ChatCompletionToolCallChunk] + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] + citationsContentBlocks: Optional[List[CitationsContentBlock]] - Citations from Nova grounding + """ + content_str = "" + tools: List[ChatCompletionToolCallChunk] = [] + reasoningContentBlocks: Optional[ + List[BedrockConverseReasoningContentBlock] + ] = None + citationsContentBlocks: Optional[List[CitationsContentBlock]] = None + for idx, content in enumerate(content_blocks): + """ + - Content is either a tool response or text + """ + extracted_reasoning_content_str: Optional[str] = None + if "text" in content: + ( + extracted_reasoning_content_str, + _content_str, + ) = _parse_content_for_reasoning(content["text"]) + if _content_str is not None: + content_str += _content_str + if "toolUse" in content: + ## check tool name was formatted by litellm + _response_tool_name = content["toolUse"]["name"] + response_tool_name = get_bedrock_tool_name( + response_tool_name=_response_tool_name + ) + _function_chunk = ChatCompletionToolCallFunctionChunk( + name=response_tool_name, + arguments=json.dumps(content["toolUse"]["input"]), + ) + + _tool_response_chunk = ChatCompletionToolCallChunk( + id=content["toolUse"]["toolUseId"], + type="function", + function=_function_chunk, + index=idx, + ) + tools.append(_tool_response_chunk) + if extracted_reasoning_content_str is not None: + if reasoningContentBlocks is None: + reasoningContentBlocks = [] + reasoningContentBlocks.append( + BedrockConverseReasoningContentBlock( + reasoningText=BedrockConverseReasoningTextBlock( + text=extracted_reasoning_content_str, + ) + ) + ) + if "reasoningContent" in content: + if reasoningContentBlocks is None: + reasoningContentBlocks = [] + reasoningContentBlocks.append(content["reasoningContent"]) + # Handle Nova grounding citations content + if "citationsContent" in content: + if citationsContentBlocks is None: + citationsContentBlocks = [] + citationsContentBlocks.append(content["citationsContent"]) + + return content_str, tools, reasoningContentBlocks, citationsContentBlocks + + @staticmethod + def _unwrap_bedrock_properties(json_str: str) -> str: + """ + Unwrap Bedrock's response_format JSON structure. + + If the JSON has a single "properties" key, extract its value. + Otherwise, return the original string. + + Args: + json_str: JSON string to unwrap + + Returns: + Unwrapped JSON string or original if unwrapping not needed + """ + try: + response_data = json.loads(json_str) + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): + response_data = response_data["properties"] + return json.dumps(response_data) + except json.JSONDecodeError: + pass + return json_str + + @staticmethod + def _filter_json_mode_tools( + json_mode: Optional[bool], + tools: List[ChatCompletionToolCallChunk], + chat_completion_message: ChatCompletionResponseMessage, + ) -> Optional[List[ChatCompletionToolCallChunk]]: + """ + When json_mode is True, Bedrock may return the internal `json_tool_call` + tool alongside real user-defined tools. This method handles 3 scenarios: + + 1. Only json_tool_call present -> convert to text content, return None + 2. Mixed json_tool_call + real -> filter out json_tool_call, return real tools + 3. No json_tool_call / no json_mode -> return tools as-is + """ + if not json_mode or not tools: + return tools if tools else None + + json_tool_indices = [ + i + for i, t in enumerate(tools) + if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME + ] + + if not json_tool_indices: + # No json_tool_call found, return tools unchanged + return tools + + if len(json_tool_indices) == len(tools): + # All tools are json_tool_call — convert first one to content + verbose_logger.debug( + "Processing JSON tool call response for response_format" + ) + json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments") + if json_mode_content_str is not None: + json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties( + json_mode_content_str + ) + chat_completion_message["content"] = json_mode_content_str + return None + + # Mixed: filter out json_tool_call, keep real tools. + # Preserve the json_tool_call content as message text so the structured + # output from response_format is not silently lost. + first_idx = json_tool_indices[0] + json_mode_args = tools[first_idx]["function"].get("arguments") + if json_mode_args is not None: + json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties( + json_mode_args + ) + existing = chat_completion_message.get("content") or "" + chat_completion_message["content"] = ( + existing + json_mode_args if existing else json_mode_args + ) + + real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices] + return real_tools if real_tools else None + + def _transform_response( # noqa: PLR0915 + self, + model: str, + response: httpx.Response, + model_response: ModelResponse, + stream: bool, + logging_obj: Optional[Logging], + optional_params: dict, + api_key: Optional[str], + data: Union[dict, str], + messages: List, + encoding, + ) -> ModelResponse: + ## LOGGING + if logging_obj is not None: + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + + json_mode: Optional[bool] = optional_params.get("json_mode", None) + ## RESPONSE OBJECT + try: + completion_response = ConverseResponseBlock(**response.json()) # type: ignore + except Exception as e: + raise BedrockError( + message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( + response.text, str(e) + ), + status_code=422, + ) + + """ + Bedrock Response Object has optional message block + + completion_response["output"].get("message", None) + + A message block looks like this (Example 1): + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "Is there anything else you'd like to talk about? Perhaps I can help with some economic questions or provide some information about economic concepts?" + } + ] + } + }, + (Example 2): + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_hbTgdi0CSLq_hM4P8csZJA", + "name": "top_song", + "input": { + "sign": "WZPZ" + } + } + } + ] + } + } + + """ + message: Optional[MessageBlock] = completion_response["output"]["message"] + chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} + content_str = "" + tools: List[ChatCompletionToolCallChunk] = [] + reasoningContentBlocks: Optional[ + List[BedrockConverseReasoningContentBlock] + ] = None + citationsContentBlocks: Optional[List[CitationsContentBlock]] = None + + if message is not None: + ( + content_str, + tools, + reasoningContentBlocks, + citationsContentBlocks, + ) = self._translate_message_content(message["content"]) + + # Initialize provider_specific_fields if we have any special content blocks + provider_specific_fields: dict = {} + if reasoningContentBlocks is not None: + provider_specific_fields["reasoningContentBlocks"] = reasoningContentBlocks + if citationsContentBlocks is not None: + provider_specific_fields["citationsContent"] = citationsContentBlocks + + if provider_specific_fields: + chat_completion_message[ + "provider_specific_fields" + ] = provider_specific_fields + + if reasoningContentBlocks is not None: + chat_completion_message[ + "reasoning_content" + ] = self._transform_reasoning_content(reasoningContentBlocks) + chat_completion_message[ + "thinking_blocks" + ] = self._transform_thinking_blocks(reasoningContentBlocks) + chat_completion_message["content"] = content_str + filtered_tools = self._filter_json_mode_tools( + json_mode=json_mode, + tools=tools, + chat_completion_message=chat_completion_message, + ) + if filtered_tools: + chat_completion_message["tool_calls"] = filtered_tools + + ## CALCULATING USAGE - bedrock returns usage in the headers + usage = self._transform_usage( + completion_response["usage"], + reasoning_content=chat_completion_message.get("reasoning_content"), + ) + + ## HANDLE TOOL CALLS + _message = Message(**chat_completion_message) + initial_finish_reason = map_finish_reason(completion_response["stopReason"]) + + ( + returned_message, + returned_finish_reason, + ) = self.apply_tool_call_transformation_if_needed( + message=_message, + tools=optional_params.get("tools"), + initial_finish_reason=initial_finish_reason, + ) + model_response.choices = [ + litellm.Choices( + finish_reason=returned_finish_reason, + index=0, + message=returned_message, + ) + ] + model_response.created = int(time.time()) + model_response.model = model + + setattr(model_response, "usage", usage) + + # Add "trace" from Bedrock guardrails - if user has opted in to returning it + if "trace" in completion_response: + setattr(model_response, "trace", completion_response["trace"]) + + # Add service_tier if present in Bedrock response + # Map Bedrock serviceTier (object) to OpenAI service_tier (string) + if "serviceTier" in completion_response: + service_tier_block = completion_response["serviceTier"] + if isinstance(service_tier_block, dict) and "type" in service_tier_block: + setattr(model_response, "service_tier", service_tier_block["type"]) + + return model_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return BedrockError( + message=error_message, + status_code=status_code, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + fake_stream: Optional[bool] = None, + ) -> bool: + """ + Returns True if the model/provider should fake stream + """ + ################################################################### + # If an upstream method already set fake_stream to True, return True + ################################################################### + if fake_stream is True: + return True + + ################################################################### + # Bedrock Converse Specific Logic + ################################################################### + if stream is True: + if model is not None: + ################################################################### + # AI21 models do not support streaming + ################################################################### + if "ai21" in model: + return True + return False diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_agent/transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..2c7135f4d83576394aaf2b12a20f1afc289e3292 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -0,0 +1,547 @@ +""" +Transformation for Bedrock Invoke Agent + +https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html +""" + +import base64 +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import httpx + +from litellm._logging import verbose_logger +from litellm._uuid import uuid +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, +) +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.types.llms.bedrock_invoke_agents import ( + InvokeAgentChunkPayload, + InvokeAgentEvent, + InvokeAgentEventHeaders, + InvokeAgentEventList, + InvokeAgentMetadata, + InvokeAgentModelInvocationInput, + InvokeAgentModelInvocationOutput, + InvokeAgentOrchestrationTrace, + InvokeAgentPreProcessingTrace, + InvokeAgentTrace, + InvokeAgentTracePayload, + InvokeAgentUsage, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): + def __init__(self, **kwargs): + BaseConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + def get_supported_openai_params(self, model: str) -> List[str]: + """ + This is a base invoke agent model mapping. For Invoke Agent - define a bedrock provider specific config that extends this class. + + Bedrock Invoke Agents has 0 OpenAI compatible params + + As of May 29th, 2025 - they don't support streaming. + """ + return [] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + This is a base invoke agent model mapping. For Invoke Agent - define a bedrock provider specific config that extends this class. + """ + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete url for the request + """ + ### SET RUNTIME ENDPOINT ### + aws_bedrock_runtime_endpoint = optional_params.get( + "aws_bedrock_runtime_endpoint", None + ) # https://bedrock-runtime.{region_name}.amazonaws.com + endpoint_url, _ = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_region_name=self._get_aws_region_name( + optional_params=optional_params, model=model + ), + endpoint_type="agent", + ) + + agent_id, agent_alias_id = self._get_agent_id_and_alias_id(model) + session_id = self._get_session_id(optional_params) + + endpoint_url = f"{endpoint_url}/agents/{agent_id}/agentAliases/{agent_alias_id}/sessions/{session_id}/text" + + return endpoint_url + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + model=model, + stream=stream, + fake_stream=fake_stream, + api_key=api_key, + ) + + def _get_agent_id_and_alias_id(self, model: str) -> tuple[str, str]: + """ + model = "agent/L1RT58GYRW/MFPSBCXYTW" + agent_id = "L1RT58GYRW" + agent_alias_id = "MFPSBCXYTW" + """ + # Split the model string by '/' and extract components + parts = model.split("/") + if len(parts) != 3 or parts[0] != "agent": + raise ValueError( + "Invalid model format. Expected format: 'model=agent/AGENT_ID/ALIAS_ID'" + ) + + return parts[1], parts[2] # Return (agent_id, agent_alias_id) + + def _get_session_id(self, optional_params: dict) -> str: + """ """ + return optional_params.get("sessionID", None) or str(uuid.uuid4()) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + # use the last message content as the query + query: str = convert_content_list_to_str(messages[-1]) + return { + "inputText": query, + "enableTrace": True, + **optional_params, + } + + def _parse_aws_event_stream(self, raw_content: bytes) -> InvokeAgentEventList: + """ + Parse AWS event stream format using boto3/botocore's built-in parser. + This is the same approach used in the existing AWSEventStreamDecoder. + """ + try: + from botocore.eventstream import EventStreamBuffer + from botocore.parsers import EventStreamJSONParser + except ImportError: + raise ImportError("boto3/botocore is required for AWS event stream parsing") + + events: InvokeAgentEventList = [] + parser = EventStreamJSONParser() + event_stream_buffer = EventStreamBuffer() + + # Add the entire response to the buffer + event_stream_buffer.add_data(raw_content) + + # Process all events in the buffer + for event in event_stream_buffer: + try: + headers = self._extract_headers_from_event(event) + + event_type = headers.get("event_type", "") + + if event_type == "chunk": + # Handle chunk events specially - they contain decoded content, not JSON + message = self._parse_message_from_event(event, parser) + parsed_event: InvokeAgentEvent = InvokeAgentEvent() + if message: + # For chunk events, create a payload with the decoded content + parsed_event = { + "headers": headers, + "payload": { + "bytes": base64.b64encode( + message.encode("utf-8") + ).decode("utf-8") + }, # Re-encode for consistency + } + events.append(parsed_event) + + elif event_type == "trace": + # Handle trace events normally - they contain JSON + message = self._parse_message_from_event(event, parser) + + if message: + try: + event_data = json.loads(message) + parsed_event = { + "headers": headers, + "payload": event_data, + } + events.append(parsed_event) + except json.JSONDecodeError as e: + verbose_logger.warning( + f"Failed to parse trace event JSON: {e}" + ) + else: + verbose_logger.debug(f"Unknown event type: {event_type}") + + except Exception as e: + verbose_logger.error(f"Error processing event: {e}") + continue + + return events + + def _parse_message_from_event(self, event, parser) -> Optional[str]: + """Extract message content from an AWS event, adapted from AWSEventStreamDecoder.""" + try: + response_dict = event.to_response_dict() + verbose_logger.debug(f"Response dict: {response_dict}") + + # Use the same response shape parsing as the existing decoder + parsed_response = parser.parse( + response_dict, self._get_response_stream_shape() + ) + verbose_logger.debug(f"Parsed response: {parsed_response}") + + if response_dict["status_code"] != 200: + decoded_body = response_dict["body"].decode() + if isinstance(decoded_body, dict): + error_message = decoded_body.get("message") + elif isinstance(decoded_body, str): + error_message = decoded_body + else: + error_message = "" + exception_status = response_dict["headers"].get(":exception-type") + error_message = exception_status + " " + error_message + raise BedrockError( + status_code=response_dict["status_code"], + message=( + json.dumps(error_message) + if isinstance(error_message, dict) + else error_message + ), + ) + + if "chunk" in parsed_response: + chunk = parsed_response.get("chunk") + if not chunk: + return None + return chunk.get("bytes").decode() + else: + chunk = response_dict.get("body") + if not chunk: + return None + return chunk.decode() + + except Exception as e: + verbose_logger.debug(f"Error parsing message from event: {e}") + return None + + def _extract_headers_from_event(self, event) -> InvokeAgentEventHeaders: + """Extract headers from an AWS event for categorization.""" + try: + response_dict = event.to_response_dict() + headers = response_dict.get("headers", {}) + + # Extract the event-type and content-type headers that we care about + return InvokeAgentEventHeaders( + event_type=headers.get(":event-type", ""), + content_type=headers.get(":content-type", ""), + message_type=headers.get(":message-type", ""), + ) + except Exception as e: + verbose_logger.debug(f"Error extracting headers: {e}") + return InvokeAgentEventHeaders( + event_type="", content_type="", message_type="" + ) + + def _get_response_stream_shape(self): + """Get the response stream shape for parsing, reusing existing logic.""" + try: + # Try to reuse the cached shape from the existing decoder + from litellm.llms.bedrock.chat.invoke_handler import ( + get_response_stream_shape, + ) + + return get_response_stream_shape() + except ImportError: + # Fallback: create our own shape + try: + from botocore.loaders import Loader + from botocore.model import ServiceModel + + loader = Loader() + bedrock_service_dict = loader.load_service_model( + "bedrock-runtime", "service-2" + ) + bedrock_service_model = ServiceModel(bedrock_service_dict) + return bedrock_service_model.shape_for("ResponseStream") + except Exception as e: + verbose_logger.warning(f"Could not load response stream shape: {e}") + return None + + def _extract_response_content(self, events: InvokeAgentEventList) -> str: + """Extract the final response content from parsed events.""" + response_parts = [] + + for event in events: + headers = event.get("headers", {}) + payload = event.get("payload") + + event_type = headers.get( + "event_type" + ) # Note: using event_type not event-type + + if event_type == "chunk" and payload: + # Extract base64 encoded content from chunk events + chunk_payload: InvokeAgentChunkPayload = payload # type: ignore + encoded_bytes = chunk_payload.get("bytes", "") + if encoded_bytes: + try: + decoded_content = base64.b64decode(encoded_bytes).decode( + "utf-8" + ) + response_parts.append(decoded_content) + except Exception as e: + verbose_logger.warning(f"Failed to decode chunk content: {e}") + + return "".join(response_parts) + + def _extract_usage_info(self, events: InvokeAgentEventList) -> InvokeAgentUsage: + """Extract token usage information from trace events.""" + usage_info = InvokeAgentUsage( + inputTokens=0, + outputTokens=0, + model=None, + ) + + response_model: Optional[str] = None + + for event in events: + if not self._is_trace_event(event): + continue + + trace_data = self._get_trace_data(event) + if not trace_data: + continue + + verbose_logger.debug(f"Trace event: {trace_data}") + + # Extract usage from pre-processing trace + self._extract_and_update_preprocessing_usage( + trace_data=trace_data, + usage_info=usage_info, + ) + + # Extract model from orchestration trace + if response_model is None: + response_model = self._extract_orchestration_model(trace_data) + + usage_info["model"] = response_model + return usage_info + + def _is_trace_event(self, event: InvokeAgentEvent) -> bool: + """Check if the event is a trace event.""" + headers = event.get("headers", {}) + event_type = headers.get("event_type") + payload = event.get("payload") + return event_type == "trace" and payload is not None + + def _get_trace_data(self, event: InvokeAgentEvent) -> Optional[InvokeAgentTrace]: + """Extract trace data from a trace event.""" + payload = event.get("payload") + if not payload: + return None + + trace_payload: InvokeAgentTracePayload = payload # type: ignore + return trace_payload.get("trace", {}) + + def _extract_and_update_preprocessing_usage( + self, trace_data: InvokeAgentTrace, usage_info: InvokeAgentUsage + ) -> None: + """Extract usage information from preprocessing trace.""" + pre_processing: Optional[InvokeAgentPreProcessingTrace] = trace_data.get( + "preProcessingTrace" + ) + if not pre_processing: + return + + model_output: Optional[InvokeAgentModelInvocationOutput] = ( + pre_processing.get("modelInvocationOutput") + or InvokeAgentModelInvocationOutput() + ) + if not model_output: + return + + metadata: Optional[InvokeAgentMetadata] = ( + model_output.get("metadata") or InvokeAgentMetadata() + ) + if not metadata: + return + + usage: Optional[Union[InvokeAgentUsage, Dict]] = metadata.get("usage", {}) + if not usage: + return + + usage_info["inputTokens"] += usage.get("inputTokens", 0) + usage_info["outputTokens"] += usage.get("outputTokens", 0) + + def _extract_orchestration_model( + self, trace_data: InvokeAgentTrace + ) -> Optional[str]: + """Extract model information from orchestration trace.""" + orchestration_trace: Optional[InvokeAgentOrchestrationTrace] = trace_data.get( + "orchestrationTrace" + ) + if not orchestration_trace: + return None + + model_invocation: Optional[InvokeAgentModelInvocationInput] = ( + orchestration_trace.get("modelInvocationInput") + or InvokeAgentModelInvocationInput() + ) + if not model_invocation: + return None + + return model_invocation.get("foundationModel") + + def _build_model_response( + self, + content: str, + model: str, + usage_info: InvokeAgentUsage, + model_response: ModelResponse, + ) -> ModelResponse: + """Build the final ModelResponse object.""" + + # Create the message content + message = Message(content=content, role="assistant") + + # Create choices + choice = Choices(finish_reason="stop", index=0, message=message) + + # Update model response + model_response.choices = [choice] + model_response.model = usage_info.get("model", model) + + # Add usage information if available + if usage_info: + from litellm.types.utils import Usage + + usage = Usage( + prompt_tokens=usage_info.get("inputTokens", 0), + completion_tokens=usage_info.get("outputTokens", 0), + total_tokens=usage_info.get("inputTokens", 0) + + usage_info.get("outputTokens", 0), + ) + setattr(model_response, "usage", usage) + + return model_response + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + try: + # Get the raw binary content + raw_content = raw_response.content + verbose_logger.debug( + f"Processing {len(raw_content)} bytes of AWS event stream data" + ) + + # Parse the AWS event stream format + events = self._parse_aws_event_stream(raw_content) + verbose_logger.debug(f"Parsed {len(events)} events from stream") + + # Extract response content from chunk events + content = self._extract_response_content(events) + + # Extract usage information from trace events + usage_info = self._extract_usage_info(events) + + # Build and return the model response + return self._build_model_response( + content=content, + model=model, + usage_info=usage_info, + model_response=model_response, + ) + + except Exception as e: + verbose_logger.error( + f"Error processing Bedrock Invoke Agent response: {str(e)}" + ) + raise BedrockError( + message=f"Error processing response: {str(e)}", + status_code=raw_response.status_code, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return BedrockError(status_code=status_code, message=error_message) + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + return True diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..1077731779dc16a7beafe6922363a1c57bcfceba --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_handler.py @@ -0,0 +1,1893 @@ +""" +TODO: DELETE FILE. Bedrock LLM is no longer used. Goto `litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py` +""" + +import copy +import time +import types +from functools import partial +from typing import ( + AsyncIterator, + Callable, + Iterator, + Optional, + Tuple, + cast, + get_args, +) + +import httpx # type: ignore + +import litellm +from litellm import verbose_logger +from litellm._uuid import uuid +from litellm.caching.caching import InMemoryCache +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.logging_utils import track_llm_api_timing +from litellm.litellm_core_utils.prompt_templates.factory import ( + cohere_message_pt, + construct_tool_use_system_prompt, + contains_tag, + custom_prompt, + extract_between_tags, + parse_xml_params, + prompt_factory, +) +from litellm.llms.anthropic.chat.handler import ( + ModelResponseIterator as AnthropicModelResponseIterator, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.types.llms.bedrock import * +from litellm.types.llms.openai import ( + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, + ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, + ChatCompletionUsageBlock, +) +from litellm.types.utils import ( + ChatCompletionMessageToolCall, + Choices, + Delta, +) +from litellm.types.utils import GenericStreamingChunk as GChunk +from litellm.types.utils import ( + ModelResponse, + ModelResponseStream, + StreamingChoices, + Usage, +) +from litellm.utils import CustomStreamWrapper, get_secret + +from ..base_aws_llm import BaseAWSLLM +from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name + +_response_stream_shape_cache = None +bedrock_tool_name_mappings: InMemoryCache = InMemoryCache( + max_size_in_memory=50, default_ttl=600 +) +from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, +) + +converse_config = AmazonConverseConfig() + + +class AmazonCohereChatConfig: + """ + Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command-r-plus.html + """ + + documents: Optional[List[Document]] = None + search_queries_only: Optional[bool] = None + preamble: Optional[str] = None + max_tokens: Optional[int] = None + temperature: Optional[float] = None + p: Optional[float] = None + k: Optional[float] = None + prompt_truncation: Optional[str] = None + frequency_penalty: Optional[float] = None + presence_penalty: Optional[float] = None + seed: Optional[int] = None + return_prompt: Optional[bool] = None + stop_sequences: Optional[List[str]] = None + raw_prompting: Optional[bool] = None + + def __init__( + self, + documents: Optional[List[Document]] = None, + search_queries_only: Optional[bool] = None, + preamble: Optional[str] = None, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + p: Optional[float] = None, + k: Optional[float] = None, + prompt_truncation: Optional[str] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + seed: Optional[int] = None, + return_prompt: Optional[bool] = None, + stop_sequences: Optional[str] = None, + raw_prompting: Optional[bool] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self) -> List[str]: + return [ + "max_tokens", + "max_completion_tokens", + "stream", + "stop", + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + "seed", + "stop", + "tools", + "tool_choice", + ] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + for param, value in non_default_params.items(): + if param == "max_tokens" or param == "max_completion_tokens": + optional_params["max_tokens"] = value + if param == "stream": + optional_params["stream"] = value + if param == "stop": + if isinstance(value, str): + value = [value] + optional_params["stop_sequences"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["p"] = value + if param == "frequency_penalty": + optional_params["frequency_penalty"] = value + if param == "presence_penalty": + optional_params["presence_penalty"] = value + if "seed": + optional_params["seed"] = value + return optional_params + + +async def make_call( + client: Optional[AsyncHTTPHandler], + api_base: str, + headers: dict, + data: str, + model: str, + messages: list, + logging_obj: Logging, + fake_stream: bool = False, + json_mode: Optional[bool] = False, + bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, + stream_chunk_size: int = 1024, +): + try: + if client is None: + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.BEDROCK, + params={"ssl_verify": logging_obj.litellm_params.get("ssl_verify")} + if logging_obj + and logging_obj.litellm_params + and logging_obj.litellm_params.get("ssl_verify") + else None, + ) # Create a new client if none provided + + response = await client.post( + api_base, + headers=headers, + data=data, + stream=not fake_stream, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise BedrockError(status_code=response.status_code, message=response.text) + + if fake_stream: + model_response: ( + ModelResponse + ) = litellm.AmazonConverseConfig()._transform_response( + model=model, + response=response, + model_response=litellm.ModelResponse(), + stream=True, + logging_obj=logging_obj, + optional_params={}, + api_key="", + data=data, + messages=messages, + encoding=litellm.encoding, + ) # type: ignore + completion_stream: Any = MockResponseIterator( + model_response=model_response, json_mode=json_mode + ) + elif bedrock_invoke_provider == "anthropic": + decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder( + model=model, + sync_stream=False, + json_mode=json_mode, + ) + completion_stream = decoder.aiter_bytes( + response.aiter_bytes(chunk_size=stream_chunk_size) + ) + elif bedrock_invoke_provider == "deepseek_r1": + decoder = AmazonDeepSeekR1StreamDecoder( + model=model, + sync_stream=False, + ) + completion_stream = decoder.aiter_bytes( + response.aiter_bytes(chunk_size=stream_chunk_size) + ) + else: + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) + completion_stream = decoder.aiter_bytes( + response.aiter_bytes(chunk_size=stream_chunk_size) + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return completion_stream + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=err.response.text) + except httpx.TimeoutException: + raise BedrockError(status_code=408, message="Timeout error occurred.") + except Exception as e: + raise BedrockError(status_code=500, message=str(e)) + + +def make_sync_call( + client: Optional[HTTPHandler], + api_base: str, + headers: dict, + data: str, + signed_json_body: Optional[bytes], + model: str, + messages: list, + logging_obj: Logging, + fake_stream: bool = False, + json_mode: Optional[bool] = False, + bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, + stream_chunk_size: int = 1024, +): + try: + if client is None: + client = _get_httpx_client( + params={"ssl_verify": logging_obj.litellm_params.get("ssl_verify")} + if logging_obj + and logging_obj.litellm_params + and logging_obj.litellm_params.get("ssl_verify") + else None + ) + + response = client.post( + api_base, + headers=headers, + data=signed_json_body if signed_json_body is not None else data, + stream=not fake_stream, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise BedrockError(status_code=response.status_code, message=response.text) + + if fake_stream: + model_response: ( + ModelResponse + ) = litellm.AmazonConverseConfig()._transform_response( + model=model, + response=response, + model_response=litellm.ModelResponse(), + stream=True, + logging_obj=logging_obj, + optional_params={}, + api_key="", + data=data, + messages=messages, + encoding=litellm.encoding, + ) # type: ignore + completion_stream: Any = MockResponseIterator( + model_response=model_response, json_mode=json_mode + ) + elif bedrock_invoke_provider == "anthropic": + decoder: AWSEventStreamDecoder = AmazonAnthropicClaudeStreamDecoder( + model=model, + sync_stream=True, + json_mode=json_mode, + ) + completion_stream = decoder.iter_bytes( + response.iter_bytes(chunk_size=stream_chunk_size) + ) + elif bedrock_invoke_provider == "deepseek_r1": + decoder = AmazonDeepSeekR1StreamDecoder( + model=model, + sync_stream=True, + ) + completion_stream = decoder.iter_bytes( + response.iter_bytes(chunk_size=stream_chunk_size) + ) + else: + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) + completion_stream = decoder.iter_bytes( + response.iter_bytes(chunk_size=stream_chunk_size) + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return completion_stream + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=err.response.text) + except httpx.TimeoutException: + raise BedrockError(status_code=408, message="Timeout error occurred.") + except Exception as e: + raise BedrockError(status_code=500, message=str(e)) + + +class BedrockLLM(BaseAWSLLM): + """ + Example call + + ``` + curl --location --request POST 'https://bedrock-runtime.{aws_region_name}.amazonaws.com/model/{bedrock_model_name}/invoke' \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + --user "$AWS_ACCESS_KEY_ID":"$AWS_SECRET_ACCESS_KEY" \ + --aws-sigv4 "aws:amz:us-east-1:bedrock" \ + --data-raw '{ + "prompt": "Hi", + "temperature": 0, + "p": 0.9, + "max_tokens": 4096 + }' + ``` + """ + + def __init__(self) -> None: + super().__init__() + + @staticmethod + def is_claude_messages_api_model(model: str) -> bool: + """ + Check if the model uses the Claude Messages API (Claude 3+). + + Handles: + - Regional prefixes: eu.anthropic.claude-*, us.anthropic.claude-* + - Claude 3 models: claude-3-haiku, claude-3-sonnet, claude-3-opus, claude-3-5-*, claude-3-7-* + - Claude 4 models: claude-opus-4, claude-sonnet-4, claude-haiku-4 + """ + # Normalize model string to lowercase for matching + model_lower = model.lower() + + # Claude 3+ indicators (all use Messages API) + messages_api_indicators = [ + "claude-3", # Claude 3.x models + "claude-opus-4", # Claude Opus 4 + "claude-sonnet-4", # Claude Sonnet 4 + "claude-haiku-4", # Claude Haiku 4 + ] + + return any(indicator in model_lower for indicator in messages_api_indicators) + + def convert_messages_to_prompt( + self, model, messages, provider, custom_prompt_dict + ) -> Tuple[str, Optional[list]]: + # handle anthropic prompts and amazon titan prompts + prompt = "" + chat_history: Optional[list] = None + ## CUSTOM PROMPT + if model in custom_prompt_dict: + # check if the model has a registered custom prompt + model_prompt_details = custom_prompt_dict[model] + prompt = custom_prompt( + role_dict=model_prompt_details["roles"], + initial_prompt_value=model_prompt_details.get( + "initial_prompt_value", "" + ), + final_prompt_value=model_prompt_details.get("final_prompt_value", ""), + messages=messages, + ) + return prompt, None + ## ELSE + if provider == "anthropic" or provider == "amazon": + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "mistral": + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "meta" or provider == "llama": + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "openai": + # OpenAI uses messages directly, no prompt conversion needed + # Return empty prompt as it won't be used + prompt = "" + elif provider == "cohere": + prompt, chat_history = cohere_message_pt(messages=messages) + else: + prompt = "" + for message in messages: + if "role" in message: + if message["role"] == "user": + prompt += f"{message['content']}" + else: + prompt += f"{message['content']}" + else: + prompt += f"{message['content']}" + return prompt, chat_history # type: ignore + + def process_response( # noqa: PLR0915 + self, + model: str, + response: httpx.Response, + model_response: ModelResponse, + stream: Optional[bool], + logging_obj: Logging, + optional_params: dict, + api_key: str, + data: Union[dict, str], + messages: List, + print_verbose, + encoding, + ) -> Union[ModelResponse, CustomStreamWrapper]: + provider = self.get_bedrock_invoke_provider(model) + ## LOGGING + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + print_verbose(f"raw model_response: {response.text}") + + ## RESPONSE OBJECT + try: + completion_response = response.json() + except Exception: + raise BedrockError(message=response.text, status_code=422) + + outputText: Optional[str] = None + try: + if provider == "cohere": + if "text" in completion_response: + outputText = completion_response["text"] # type: ignore + elif "generations" in completion_response: + outputText = completion_response["generations"][0]["text"] + model_response.choices[0].finish_reason = map_finish_reason( + completion_response["generations"][0]["finish_reason"] + ) + elif provider == "anthropic": + if self.is_claude_messages_api_model(model): + json_schemas: dict = {} + _is_function_call = False + ## Handle Tool Calling + if "tools" in optional_params: + _is_function_call = True + for tool in optional_params["tools"]: + json_schemas[tool["function"]["name"]] = tool[ + "function" + ].get("parameters", None) + outputText = completion_response.get("content")[0].get("text", None) + if outputText is not None and contains_tag( + "invoke", outputText + ): # OUTPUT PARSE FUNCTION CALL + function_name = extract_between_tags("tool_name", outputText)[0] + function_arguments_str = extract_between_tags( + "invoke", outputText + )[0].strip() + function_arguments_str = ( + f"{function_arguments_str}" + ) + function_arguments = parse_xml_params( + function_arguments_str, + json_schema=json_schemas.get( + function_name, None + ), # check if we have a json schema for this function name) + ) + _message = litellm.Message( + tool_calls=[ + { + "id": f"call_{uuid.uuid4()}", + "type": "function", + "function": { + "name": function_name, + "arguments": json.dumps(function_arguments), + }, + } + ], + content=None, + ) + model_response.choices[0].message = _message # type: ignore + model_response._hidden_params[ + "original_response" + ] = outputText # allow user to access raw anthropic tool calling response + if ( + _is_function_call is True + and stream is not None + and stream is True + ): + print_verbose( + "INSIDE BEDROCK STREAMING TOOL CALLING CONDITION BLOCK" + ) + # return an iterator + streaming_model_response = ModelResponseStream() + streaming_model_response.choices[0].finish_reason = getattr( + model_response.choices[0], "finish_reason", "stop" + ) + # streaming_model_response.choices = [litellm.utils.StreamingChoices()] + streaming_choice = litellm.utils.StreamingChoices() + streaming_choice.index = model_response.choices[0].index + _tool_calls = [] + print_verbose( + f"type of model_response.choices[0]: {type(model_response.choices[0])}" + ) + print_verbose( + f"type of streaming_choice: {type(streaming_choice)}" + ) + if isinstance(model_response.choices[0], litellm.Choices): + if getattr( + model_response.choices[0].message, "tool_calls", None + ) is not None and isinstance( + model_response.choices[0].message.tool_calls, list + ): + for tool_call in model_response.choices[ + 0 + ].message.tool_calls: + _tool_call = {**tool_call.dict(), "index": 0} + _tool_calls.append(_tool_call) + delta_obj = Delta( + content=getattr( + model_response.choices[0].message, "content", None + ), + role=model_response.choices[0].message.role, + tool_calls=_tool_calls, + ) + streaming_choice.delta = delta_obj + streaming_model_response.choices = [streaming_choice] + completion_stream = ModelResponseIterator( + model_response=streaming_model_response + ) + print_verbose( + "Returns anthropic CustomStreamWrapper with 'cached_response' streaming object" + ) + return litellm.CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="cached_response", + logging_obj=logging_obj, + ) + + model_response.choices[0].finish_reason = map_finish_reason( + completion_response.get("stop_reason", "") + ) + _usage = litellm.Usage( + prompt_tokens=completion_response["usage"]["input_tokens"], + completion_tokens=completion_response["usage"]["output_tokens"], + total_tokens=completion_response["usage"]["input_tokens"] + + completion_response["usage"]["output_tokens"], + ) + setattr(model_response, "usage", _usage) + else: + outputText = completion_response["completion"] + + model_response.choices[0].finish_reason = completion_response[ + "stop_reason" + ] + elif provider == "ai21": + outputText = ( + completion_response.get("completions")[0].get("data").get("text") + ) + elif provider == "meta" or provider == "llama": + outputText = completion_response["generation"] + elif provider == "openai": + # OpenAI imported models use OpenAI Chat Completions format + if ( + "choices" in completion_response + and len(completion_response["choices"]) > 0 + ): + choice = completion_response["choices"][0] + if "message" in choice: + outputText = choice["message"].get("content") + elif "text" in choice: # fallback for completion format + outputText = choice["text"] + + # Set finish reason + if "finish_reason" in choice: + model_response.choices[0].finish_reason = map_finish_reason( + choice["finish_reason"] + ) + + # Set usage if available + if "usage" in completion_response: + usage = completion_response["usage"] + _usage = litellm.Usage( + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + total_tokens=usage.get("total_tokens", 0), + ) + setattr(model_response, "usage", _usage) + elif provider == "mistral": + outputText = completion_response["outputs"][0]["text"] + model_response.choices[0].finish_reason = completion_response[ + "outputs" + ][0]["stop_reason"] + else: # amazon titan + outputText = completion_response.get("results")[0].get("outputText") + except Exception as e: + raise BedrockError( + message="Error processing={}, Received error={}".format( + response.text, str(e) + ), + status_code=422, + ) + + try: + if ( + outputText is not None + and len(outputText) > 0 + and hasattr(model_response.choices[0], "message") + and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore + is None + ): + model_response.choices[0].message.content = outputText # type: ignore + elif ( + hasattr(model_response.choices[0], "message") + and getattr(model_response.choices[0].message, "tool_calls", None) # type: ignore + is not None + ): + pass + else: + raise Exception() + except Exception as e: + raise BedrockError( + message="Error parsing received text={}.\nError-{}".format( + outputText, str(e) + ), + status_code=response.status_code, + ) + + if stream and provider == "ai21": + streaming_model_response = ModelResponseStream() + streaming_model_response.choices[0].finish_reason = model_response.choices[ # type: ignore + 0 + ].finish_reason + # streaming_model_response.choices = [litellm.utils.StreamingChoices()] + streaming_choice = litellm.utils.StreamingChoices() + streaming_choice.index = model_response.choices[0].index + delta_obj = litellm.utils.Delta( + content=getattr(model_response.choices[0].message, "content", None), # type: ignore + role=model_response.choices[0].message.role, # type: ignore + ) + streaming_choice.delta = delta_obj + streaming_model_response.choices = [streaming_choice] + mri = ModelResponseIterator(model_response=streaming_model_response) + return CustomStreamWrapper( + completion_stream=mri, + model=model, + custom_llm_provider="cached_response", + logging_obj=logging_obj, + ) + + ## CALCULATING USAGE - bedrock returns usage in the headers + # Skip if usage was already set (e.g., from JSON response for OpenAI provider) + if ( + not hasattr(model_response, "usage") + or getattr(model_response, "usage", None) is None + ): + bedrock_input_tokens = response.headers.get( + "x-amzn-bedrock-input-token-count", None + ) + bedrock_output_tokens = response.headers.get( + "x-amzn-bedrock-output-token-count", None + ) + + prompt_tokens = int( + bedrock_input_tokens or litellm.token_counter(messages=messages) + ) + + completion_tokens = int( + bedrock_output_tokens + or litellm.token_counter( + text=model_response.choices[0].message.content, # type: ignore + count_response_tokens=True, + ) + ) + + model_response.created = int(time.time()) + model_response.model = model + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + setattr(model_response, "usage", usage) + else: + # Ensure created and model are set even if usage was already set + model_response.created = int(time.time()) + model_response.model = model + + return model_response + + def completion( # noqa: PLR0915 + self, + model: str, + messages: list, + api_base: Optional[str], + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj: Logging, + optional_params: dict, + acompletion: bool, + timeout: Optional[Union[float, httpx.Timeout]], + litellm_params=None, + logger_fn=None, + extra_headers: Optional[dict] = None, + client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: + try: + from botocore.credentials import Credentials + except ImportError: + raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + + ## SETUP ## + stream = optional_params.pop("stream", None) + stream_chunk_size = optional_params.pop("stream_chunk_size", 1024) + + provider = self.get_bedrock_invoke_provider(model) + modelId = self.get_bedrock_model_id( + model=model, + provider=provider, + optional_params=optional_params, + ) + + ## CREDENTIALS ## + # pop aws_secret_access_key, aws_access_key_id, aws_session_token, aws_region_name from kwargs, since completion calls fail with them + aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) + aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_session_token = optional_params.pop("aws_session_token", None) + aws_region_name = optional_params.pop("aws_region_name", None) + aws_role_name = optional_params.pop("aws_role_name", None) + aws_session_name = optional_params.pop("aws_session_name", None) + aws_profile_name = optional_params.pop("aws_profile_name", None) + aws_bedrock_runtime_endpoint = optional_params.pop( + "aws_bedrock_runtime_endpoint", None + ) # https://bedrock-runtime.{region_name}.amazonaws.com + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) + aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None) + ssl_verify = optional_params.pop("ssl_verify", None) + + ### SET REGION NAME ### + if aws_region_name is None: + # check env # + litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) + + if litellm_aws_region_name is not None and isinstance( + litellm_aws_region_name, str + ): + aws_region_name = litellm_aws_region_name + + standard_aws_region_name = get_secret("AWS_REGION", None) + if standard_aws_region_name is not None and isinstance( + standard_aws_region_name, str + ): + aws_region_name = standard_aws_region_name + + if aws_region_name is None: + aws_region_name = "us-west-2" + + credentials: Credentials = self.get_credentials( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region_name=aws_region_name, + aws_session_name=aws_session_name, + aws_profile_name=aws_profile_name, + aws_role_name=aws_role_name, + aws_web_identity_token=aws_web_identity_token, + aws_sts_endpoint=aws_sts_endpoint, + ssl_verify=ssl_verify, + ) + + ### SET RUNTIME ENDPOINT ### + endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_region_name=aws_region_name, + ) + + if (stream is not None and stream is True) and provider != "ai21": + endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream" + proxy_endpoint_url = ( + f"{proxy_endpoint_url}/model/{modelId}/invoke-with-response-stream" + ) + else: + endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" + proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" + + prompt, chat_history = self.convert_messages_to_prompt( + model, messages, provider, custom_prompt_dict + ) + inference_params = copy.deepcopy(optional_params) + json_schemas: dict = {} + if provider == "cohere": + if model.startswith("cohere.command-r"): + ## LOAD CONFIG + config = litellm.AmazonCohereChatConfig().get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + _data = {"message": prompt, **inference_params} + if chat_history is not None: + _data["chat_history"] = chat_history + data = json.dumps(_data) + else: + ## LOAD CONFIG + config = litellm.AmazonCohereConfig.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + if stream is True: + inference_params[ + "stream" + ] = True # cohere requires stream = True in inference params + data = json.dumps({"prompt": prompt, **inference_params}) + elif provider == "anthropic": + if self.is_claude_messages_api_model(model): + # Separate system prompt from rest of message + system_prompt_idx: list[int] = [] + system_messages: list[str] = [] + for idx, message in enumerate(messages): + if message["role"] == "system": + system_messages.append(message["content"]) + system_prompt_idx.append(idx) + if len(system_prompt_idx) > 0: + inference_params["system"] = "\n".join(system_messages) + messages = [ + i for j, i in enumerate(messages) if j not in system_prompt_idx + ] + # Format rest of message according to anthropic guidelines + messages = prompt_factory( + model=model, messages=messages, custom_llm_provider="anthropic_xml" + ) # type: ignore + ## LOAD CONFIG + config = litellm.AmazonAnthropicClaudeConfig.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + ## Handle Tool Calling + if "tools" in inference_params: + _is_function_call = True + for tool in inference_params["tools"]: + json_schemas[tool["function"]["name"]] = tool["function"].get( + "parameters", None + ) + tool_calling_system_prompt = construct_tool_use_system_prompt( + tools=inference_params["tools"] + ) + inference_params["system"] = ( + inference_params.get("system", "\n") + + tool_calling_system_prompt + ) # add the anthropic tool calling prompt to the system prompt + inference_params.pop("tools") + data = json.dumps({"messages": messages, **inference_params}) + else: + ## LOAD CONFIG + config = litellm.AmazonAnthropicConfig.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + data = json.dumps({"prompt": prompt, **inference_params}) + elif provider == "ai21": + ## LOAD CONFIG + config = litellm.AmazonAI21Config.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + + data = json.dumps({"prompt": prompt, **inference_params}) + elif provider == "mistral": + ## LOAD CONFIG + config = litellm.AmazonMistralConfig.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + + data = json.dumps({"prompt": prompt, **inference_params}) + elif provider == "amazon": # amazon titan + ## LOAD CONFIG + config = litellm.AmazonTitanConfig.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + + data = json.dumps( + { + "inputText": prompt, + "textGenerationConfig": inference_params, + } + ) + elif provider == "meta" or provider == "llama": + ## LOAD CONFIG + config = litellm.AmazonLlamaConfig.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + data = json.dumps({"prompt": prompt, **inference_params}) + elif provider == "openai": + ## OpenAI imported models use OpenAI Chat Completions format (messages-based) + # Use AmazonBedrockOpenAIConfig for proper OpenAI transformation + openai_config = AmazonBedrockOpenAIConfig() + supported_params = openai_config.get_supported_openai_params(model=model) + + # Filter to only supported OpenAI params + filtered_params = { + k: v for k, v in inference_params.items() if k in supported_params + } + + # OpenAI uses messages format, not prompt + data = json.dumps({"messages": messages, **filtered_params}) + else: + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": inference_params, + }, + ) + raise BedrockError( + status_code=404, + message="Bedrock Invoke HTTPX: Unknown provider={}, model={}. Try calling via converse route - `bedrock/converse/`.".format( + provider, model + ), + ) + + ## COMPLETION CALL + + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + prepped = self.get_request_headers( + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=endpoint_url, + data=data, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": proxy_endpoint_url, + "headers": prepped.headers, + }, + ) + + ### ROUTING (ASYNC, STREAMING, SYNC) + if acompletion: + if isinstance(client, HTTPHandler): + client = None + if stream is True and provider != "ai21": + return self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=True, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) # type: ignore + ### ASYNC COMPLETION + return self.async_completion( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, # type: ignore + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) # type: ignore + + if client is None or isinstance(client, AsyncHTTPHandler): + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + self.client = _get_httpx_client(_params) # type: ignore + else: + self.client = client + if (stream is not None and stream is True) and provider != "ai21": + response = self.client.post( + url=proxy_endpoint_url, + headers=prepped.headers, # type: ignore + data=data, + stream=stream, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise BedrockError( + status_code=response.status_code, message=str(response.read()) + ) + + decoder = AWSEventStreamDecoder(model=model) + + completion_stream = decoder.iter_bytes( + response.iter_bytes(chunk_size=stream_chunk_size) + ) + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + + ## LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response=streaming_response, + additional_args={"complete_input_dict": data}, + ) + return streaming_response + + try: + response = self.client.post( + url=proxy_endpoint_url, + headers=dict(prepped.headers), + data=data, + logging_obj=logging_obj, + ) + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=err.response.text) + except httpx.TimeoutException: + raise BedrockError(status_code=408, message="Timeout error occurred.") + + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + optional_params=optional_params, + api_key="", + data=data, + messages=messages, + print_verbose=print_verbose, + encoding=encoding, + ) + + async def async_completion( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + data: str, + timeout: Optional[Union[float, httpx.Timeout]], + encoding, + logging_obj: Logging, + stream, + optional_params: dict, + litellm_params=None, + logger_fn=None, + headers={}, + client: Optional[AsyncHTTPHandler] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: + if client is None: + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + client = get_async_httpx_client(params=_params, llm_provider=litellm.LlmProviders.BEDROCK) # type: ignore + else: + client = client # type: ignore + + try: + response = await client.post( + api_base, + headers=headers, + data=data, + timeout=timeout, + logging_obj=logging_obj, + ) + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=err.response.text) + except httpx.TimeoutException: + raise BedrockError(status_code=408, message="Timeout error occurred.") + + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=stream if isinstance(stream, bool) else False, + logging_obj=logging_obj, + api_key="", + data=data, + messages=messages, + print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, + ) + + @track_llm_api_timing() # for streaming, we need to instrument the function calling the wrapper + async def async_streaming( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + data: str, + timeout: Optional[Union[float, httpx.Timeout]], + encoding, + logging_obj: Logging, + stream, + optional_params: dict, + litellm_params=None, + logger_fn=None, + headers={}, + client: Optional[AsyncHTTPHandler] = None, + stream_chunk_size: int = 1024, + ) -> CustomStreamWrapper: + # The call is not made here; instead, we prepare the necessary objects for the stream. + + streaming_response = CustomStreamWrapper( + completion_stream=None, + make_call=partial( + make_call, + client=client, + api_base=api_base, + headers=headers, + data=data, # type: ignore + model=model, + messages=messages, + logging_obj=logging_obj, + fake_stream=True if "ai21" in api_base else False, + stream_chunk_size=stream_chunk_size, + ), + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + return streaming_response + + @staticmethod + def _get_provider_from_model_path( + model_path: str, + ) -> Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL]: + """ + Helper function to get the provider from a model path with format: provider/model-name + + Args: + model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name') + + Returns: + Optional[str]: The provider name, or None if no valid provider found + """ + parts = model_path.split("/") + if len(parts) >= 1: + provider = parts[0] + if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): + return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) + return None + + +def get_response_stream_shape(): + global _response_stream_shape_cache + if _response_stream_shape_cache is None: + from botocore.loaders import Loader + from botocore.model import ServiceModel + + loader = Loader() + bedrock_service_dict = loader.load_service_model("bedrock-runtime", "service-2") + bedrock_service_model = ServiceModel(bedrock_service_dict) + _response_stream_shape_cache = bedrock_service_model.shape_for("ResponseStream") + + return _response_stream_shape_cache + + +class AWSEventStreamDecoder: + def __init__(self, model: str, json_mode: Optional[bool] = False) -> None: + from botocore.parsers import EventStreamJSONParser + + self.model = model + self.parser = EventStreamJSONParser() + self.content_blocks: List[ContentBlockDeltaEvent] = [] + self.tool_calls_index: Optional[int] = None + self.response_id: Optional[str] = None + self.json_mode = json_mode + self._current_tool_name: Optional[str] = None + + def check_empty_tool_call_args(self) -> bool: + """ + Check if the tool call block so far has been an empty string + """ + args = "" + # if text content block -> skip + if len(self.content_blocks) == 0: + return False + + if ( + "toolUse" not in self.content_blocks[0] + ): # be explicit - only do this if tool use block, as this is to prevent json decoding errors + return False + + for block in self.content_blocks: + if "toolUse" in block: + args += block["toolUse"]["input"] + + if len(args) == 0: + return True + return False + + def extract_reasoning_content_str( + self, reasoning_content_block: BedrockConverseReasoningContentBlockDelta + ) -> Optional[str]: + if "text" in reasoning_content_block: + return reasoning_content_block["text"] + return None + + def translate_thinking_blocks( + self, thinking_block: BedrockConverseReasoningContentBlockDelta + ) -> Optional[ + List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] + ]: + """ + Translate the thinking blocks to a string + """ + + thinking_blocks_list: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = [] + _thinking_block: Optional[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = None + + if "text" in thinking_block: + _thinking_block = ChatCompletionThinkingBlock(type="thinking") + _thinking_block["thinking"] = thinking_block["text"] + elif "signature" in thinking_block: + _thinking_block = ChatCompletionThinkingBlock(type="thinking") + _thinking_block["signature"] = thinking_block["signature"] + _thinking_block["thinking"] = "" # consistent with anthropic response + elif "redactedContent" in thinking_block: + _thinking_block = ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", data=thinking_block["redactedContent"] + ) + if _thinking_block is not None: + thinking_blocks_list.append(_thinking_block) + return thinking_blocks_list + + def _initialize_converse_response_id(self, chunk_data: dict): + """Initialize response_id from chunk data if not already set.""" + if self.response_id is None: + if "messageStart" in chunk_data: + conversation_id = chunk_data["messageStart"].get("conversationId") + if conversation_id: + self.response_id = f"chatcmpl-{conversation_id}" + else: + # Fallback to generating a UUID if the first chunk is not messageStart + self.response_id = f"chatcmpl-{uuid.uuid4()}" + + def _handle_converse_start_event( + self, + start_obj: ContentBlockStartEvent, + ) -> Tuple[ + Optional[ChatCompletionToolCallChunk], + dict, + Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ], + ]: + """Handle 'start' event in converse chunk parsing.""" + tool_use: Optional[ChatCompletionToolCallChunk] = None + provider_specific_fields: dict = {} + thinking_blocks: Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ] = None + + self.content_blocks = [] # reset + if start_obj is not None: + if "toolUse" in start_obj and start_obj["toolUse"] is not None: + ## check tool name was formatted by litellm + _response_tool_name = start_obj["toolUse"]["name"] + response_tool_name = get_bedrock_tool_name( + response_tool_name=_response_tool_name + ) + self._current_tool_name = response_tool_name + + # When json_mode is True, suppress the internal json_tool_call + # and convert its content to text in delta events instead + if ( + self.json_mode is True + and response_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + return tool_use, provider_specific_fields, thinking_blocks + + self.tool_calls_index = ( + 0 if self.tool_calls_index is None else self.tool_calls_index + 1 + ) + tool_use = { + "id": start_obj["toolUse"]["toolUseId"], + "type": "function", + "function": { + "name": response_tool_name, + "arguments": "", + }, + "index": self.tool_calls_index, + } + elif ( + "reasoningContent" in start_obj + and start_obj["reasoningContent"] is not None + ): # redacted thinking can be in start object + thinking_blocks = self.translate_thinking_blocks( + start_obj["reasoningContent"] + ) + provider_specific_fields = { + "reasoningContent": start_obj["reasoningContent"], + } + return tool_use, provider_specific_fields, thinking_blocks + + def _handle_converse_delta_event( + self, + delta_obj: ContentBlockDeltaEvent, + index: int, + ) -> Tuple[ + str, + Optional[ChatCompletionToolCallChunk], + dict, + Optional[str], + Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ], + ]: + """Handle 'delta' event in converse chunk parsing.""" + text = "" + tool_use: Optional[ChatCompletionToolCallChunk] = None + provider_specific_fields: dict = {} + reasoning_content: Optional[str] = None + thinking_blocks: Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ] = None + + self.content_blocks.append(delta_obj) + if "text" in delta_obj: + text = delta_obj["text"] + elif "toolUse" in delta_obj: + # When json_mode is True and this is the internal json_tool_call, + # convert tool input to text content instead of tool call arguments + if ( + self.json_mode is True + and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + text = delta_obj["toolUse"]["input"] + else: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": delta_obj["toolUse"]["input"], + }, + "index": ( + self.tool_calls_index + if self.tool_calls_index is not None + else index + ), + } + elif "reasoningContent" in delta_obj: + provider_specific_fields = { + "reasoningContent": delta_obj["reasoningContent"], + } + reasoning_content = self.extract_reasoning_content_str( + delta_obj["reasoningContent"] + ) + thinking_blocks = self.translate_thinking_blocks( + delta_obj["reasoningContent"] + ) + if ( + thinking_blocks + and len(thinking_blocks) > 0 + and reasoning_content is None + ): + reasoning_content = ( + "" # set to non-empty string to ensure consistency with Anthropic + ) + elif "citationsContent" in delta_obj: + # Handle Nova grounding citations in streaming responses + provider_specific_fields = { + "citationsContent": delta_obj["citationsContent"], + } + return ( + text, + tool_use, + provider_specific_fields, + reasoning_content, + thinking_blocks, + ) + + def _handle_converse_stop_event( + self, index: int + ) -> Optional[ChatCompletionToolCallChunk]: + """Handle stop/contentBlockIndex event in converse chunk parsing.""" + tool_use: Optional[ChatCompletionToolCallChunk] = None + + # If the ending block was the internal json_tool_call, skip emitting + # the empty-args tool chunk and reset tracking state + if ( + self.json_mode is True + and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + self._current_tool_name = None + return tool_use + + self._current_tool_name = None + is_empty = self.check_empty_tool_call_args() + if is_empty: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": "{}", + }, + "index": ( + self.tool_calls_index + if self.tool_calls_index is not None + else index + ), + } + return tool_use + + def converse_chunk_parser(self, chunk_data: dict) -> ModelResponseStream: + try: + # Capture the conversationId from the first messageStart event + # and use it as the consistent ID for all subsequent chunks. + self._initialize_converse_response_id(chunk_data) + + verbose_logger.debug("\n\nRaw Chunk: {}\n\n".format(chunk_data)) + text = "" + tool_use: Optional[ChatCompletionToolCallChunk] = None + finish_reason = "" + usage: Optional[Usage] = None + provider_specific_fields: dict = {} + reasoning_content: Optional[str] = None + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ] = None + + content_block_index = int(chunk_data.get("contentBlockIndex", 0)) + if "start" in chunk_data: + start_obj = ContentBlockStartEvent(**chunk_data["start"]) + ( + tool_use, + provider_specific_fields, + thinking_blocks, + ) = self._handle_converse_start_event(start_obj) + elif "delta" in chunk_data: + delta_obj = ContentBlockDeltaEvent(**chunk_data["delta"]) + ( + text, + tool_use, + provider_specific_fields, + reasoning_content, + thinking_blocks, + ) = self._handle_converse_delta_event(delta_obj, content_block_index) + elif ( + "contentBlockIndex" in chunk_data + ): # stop block, no 'start' or 'delta' object + tool_use = self._handle_converse_stop_event(content_block_index) + elif "stopReason" in chunk_data: + finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop")) + elif "usage" in chunk_data: + usage = converse_config._transform_usage(chunk_data.get("usage", {})) + + model_response_provider_specific_fields = {} + if "trace" in chunk_data: + trace = chunk_data.get("trace") + model_response_provider_specific_fields["trace"] = trace + response = ModelResponseStream( + choices=[ + StreamingChoices( + finish_reason=finish_reason, + index=0, # Always 0 - Bedrock never returns multiple choices + delta=Delta( + content=text, + role="assistant", + tool_calls=[tool_use] if tool_use else None, + provider_specific_fields=( + provider_specific_fields + if provider_specific_fields + else None + ), + thinking_blocks=thinking_blocks, + reasoning_content=reasoning_content, + ), + ) + ], + id=self.response_id, + model=self.model, + usage=usage, + provider_specific_fields=model_response_provider_specific_fields, + ) + + return response + except Exception as e: + raise Exception("Received streaming error - {}".format(str(e))) + + def _chunk_parser( + self, chunk_data: dict + ) -> Union[GChunk, ModelResponseStream, dict]: + text = "" + is_finished = False + finish_reason = "" + if "outputText" in chunk_data: + text = chunk_data["outputText"] + # ai21 mapping + elif "ai21" in self.model: # fake ai21 streaming + text = chunk_data.get("completions")[0].get("data").get("text") # type: ignore + is_finished = True + finish_reason = "stop" + ######## /bedrock/converse mappings ############### + elif ( + "contentBlockIndex" in chunk_data + or "stopReason" in chunk_data + or "metrics" in chunk_data + or "trace" in chunk_data + ): + return self.converse_chunk_parser(chunk_data=chunk_data) + ######### /bedrock/invoke nova mappings ############### + elif "contentBlockDelta" in chunk_data: + # when using /bedrock/invoke/nova, the chunk_data is nested under "contentBlockDelta" + _chunk_data = chunk_data.get("contentBlockDelta", {}) + return self.converse_chunk_parser(chunk_data=_chunk_data) + ######## bedrock.mistral mappings ############### + elif "outputs" in chunk_data: + if ( + len(chunk_data["outputs"]) == 1 + and chunk_data["outputs"][0].get("text", None) is not None + ): + text = chunk_data["outputs"][0]["text"] + stop_reason = chunk_data.get("stop_reason", None) + if stop_reason is not None: + is_finished = True + finish_reason = stop_reason + ######## bedrock.cohere mappings ############### + # meta mapping + elif "generation" in chunk_data: + text = chunk_data["generation"] # bedrock.meta + # cohere mapping + elif "text" in chunk_data: + text = chunk_data["text"] # bedrock.cohere + # cohere mapping for finish reason + elif "finish_reason" in chunk_data: + finish_reason = chunk_data["finish_reason"] + is_finished = True + elif chunk_data.get("completionReason", None): + is_finished = True + finish_reason = chunk_data["completionReason"] + return GChunk( + text=text, + is_finished=is_finished, + finish_reason=finish_reason, + usage=None, + index=0, + tool_use=None, + ) + + def iter_bytes( + self, iterator: Iterator[bytes] + ) -> Iterator[Union[GChunk, ModelResponseStream, dict]]: + """Given an iterator that yields lines, iterate over it & yield every event encountered""" + from botocore.eventstream import EventStreamBuffer + + event_stream_buffer = EventStreamBuffer() + for chunk in iterator: + event_stream_buffer.add_data(chunk) + for event in event_stream_buffer: + message = self._parse_message_from_event(event) + if message: + # sse_event = ServerSentEvent(data=message, event="completion") + _data = json.loads(message) + yield self._chunk_parser(chunk_data=_data) + + async def aiter_bytes( + self, iterator: AsyncIterator[bytes] + ) -> AsyncIterator[Union[GChunk, ModelResponseStream, dict]]: + """Given an async iterator that yields lines, iterate over it & yield every event encountered""" + from botocore.eventstream import EventStreamBuffer + + event_stream_buffer = EventStreamBuffer() + async for chunk in iterator: + event_stream_buffer.add_data(chunk) + for event in event_stream_buffer: + message = self._parse_message_from_event(event) + if message: + _data = json.loads(message) + yield self._chunk_parser(chunk_data=_data) + + def _parse_message_from_event(self, event) -> Optional[str]: + response_dict = event.to_response_dict() + parsed_response = self.parser.parse(response_dict, get_response_stream_shape()) + + if response_dict["status_code"] != 200: + decoded_body = response_dict["body"].decode() + if isinstance(decoded_body, dict): + error_message = decoded_body.get("message") + elif isinstance(decoded_body, str): + error_message = decoded_body + else: + error_message = "" + exception_status = response_dict["headers"].get(":exception-type") + error_message = exception_status + " " + error_message + raise BedrockError( + status_code=response_dict["status_code"], + message=( + json.dumps(error_message) + if isinstance(error_message, dict) + else error_message + ), + ) + if "chunk" in parsed_response: + chunk = parsed_response.get("chunk") + if not chunk: + return None + return chunk.get("bytes").decode() # type: ignore[no-any-return] + else: + chunk = response_dict.get("body") + if not chunk: + return None + + return chunk.decode() # type: ignore[no-any-return] + + +class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder): + def __init__( + self, + model: str, + sync_stream: bool, + json_mode: Optional[bool] = None, + ) -> None: + """ + Child class of AWSEventStreamDecoder that handles the streaming response from the Anthropic family of models + + The only difference between AWSEventStreamDecoder and AmazonAnthropicClaudeStreamDecoder is the `chunk_parser` method + """ + super().__init__(model=model) + self.anthropic_model_response_iterator = AnthropicModelResponseIterator( + streaming_response=None, + sync_stream=sync_stream, + json_mode=json_mode, + ) + + def _chunk_parser(self, chunk_data: dict) -> ModelResponseStream: + return self.anthropic_model_response_iterator.chunk_parser(chunk=chunk_data) + + +class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder): + def __init__( + self, + model: str, + sync_stream: bool, + ) -> None: + super().__init__(model=model) + from litellm.llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import ( + AmazonDeepseekR1ResponseIterator, + ) + + self.deepseek_model_response_iterator = AmazonDeepseekR1ResponseIterator( + streaming_response=None, + sync_stream=sync_stream, + ) + + def _chunk_parser( + self, chunk_data: dict + ) -> Union[GChunk, ModelResponseStream, dict]: + return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data) + + +class MockResponseIterator: # for returning ai21 streaming responses + def __init__(self, model_response, json_mode: Optional[bool] = False): + self.model_response = model_response + self.json_mode = json_mode + self.is_done = False + + # Sync iterator + def __iter__(self): + return self + + def _handle_json_mode_chunk( + self, text: str, tool_calls: Optional[List[ChatCompletionToolCallChunk]] + ) -> Tuple[str, Optional[ChatCompletionToolCallChunk]]: + """ + If JSON mode is enabled, convert the tool call to a message. + + Bedrock returns the JSON schema as part of the tool call + OpenAI returns the JSON schema as part of the content, this handles placing it in the content + + Args: + text: str + tool_use: Optional[ChatCompletionToolCallChunk] + Returns: + Tuple[str, Optional[ChatCompletionToolCallChunk]] + + text: The text to use in the content + tool_use: The ChatCompletionToolCallChunk to use in the chunk response + """ + tool_use: Optional[ChatCompletionToolCallChunk] = None + if self.json_mode is True and tool_calls is not None: + message = litellm.AnthropicConfig()._convert_tool_response_to_message( + tool_calls=tool_calls + ) + if message is not None: + text = message.content or "" + tool_use = None + elif tool_calls is not None and len(tool_calls) > 0: + tool_use = tool_calls[0] + return text, tool_use + + def _chunk_parser(self, chunk_data: ModelResponse) -> GChunk: + try: + chunk_usage: Usage = getattr(chunk_data, "usage") + text = chunk_data.choices[0].message.content or "" # type: ignore + tool_use = None + _model_response_tool_call = cast( + Optional[List[ChatCompletionMessageToolCall]], + cast(Choices, chunk_data.choices[0]).message.tool_calls, + ) + if self.json_mode is True: + text, tool_use = self._handle_json_mode_chunk( + text=text, + tool_calls=chunk_data.choices[0].message.tool_calls, # type: ignore + ) + elif _model_response_tool_call is not None: + tool_use = ChatCompletionToolCallChunk( + id=_model_response_tool_call[0].id, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=_model_response_tool_call[0].function.name, + arguments=_model_response_tool_call[0].function.arguments, + ), + index=0, + ) + processed_chunk = GChunk( + text=text, + tool_use=tool_use, + is_finished=True, + finish_reason=map_finish_reason( + finish_reason=chunk_data.choices[0].finish_reason or "" + ), + usage=ChatCompletionUsageBlock( + prompt_tokens=chunk_usage.prompt_tokens, + completion_tokens=chunk_usage.completion_tokens, + total_tokens=chunk_usage.total_tokens, + ), + index=0, + ) + return processed_chunk + except Exception as e: + raise ValueError(f"Failed to decode chunk: {chunk_data}. Error: {e}") + + def __next__(self): + if self.is_done: + raise StopIteration + self.is_done = True + return self._chunk_parser(self.model_response) + + # Async iterator + def __aiter__(self): + return self + + async def __anext__(self): + if self.is_done: + raise StopAsyncIteration + self.is_done = True + return self._chunk_parser(self.model_response) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..50fa6f170b30ec474a9c488c1d01210935e61c37 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py @@ -0,0 +1,99 @@ +import types +from typing import List, Optional + +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) + + +class AmazonAI21Config(AmazonInvokeConfig, BaseConfig): + """ + Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=j2-ultra + + Supported Params for the Amazon / AI21 models: + + - `maxTokens` (int32): The maximum number of tokens to generate per result. Optional, default is 16. If no `stopSequences` are given, generation stops after producing `maxTokens`. + + - `temperature` (float): Modifies the distribution from which tokens are sampled. Optional, default is 0.7. A value of 0 essentially disables sampling and results in greedy decoding. + + - `topP` (float): Used for sampling tokens from the corresponding top percentile of probability mass. Optional, default is 1. For instance, a value of 0.9 considers only tokens comprising the top 90% probability mass. + + - `stopSequences` (array of strings): Stops decoding if any of the input strings is generated. Optional. + + - `frequencyPenalty` (object): Placeholder for frequency penalty object. + + - `presencePenalty` (object): Placeholder for presence penalty object. + + - `countPenalty` (object): Placeholder for count penalty object. + """ + + maxTokens: Optional[int] = None + temperature: Optional[float] = None + topP: Optional[float] = None + stopSequences: Optional[list] = None + frequencePenalty: Optional[dict] = None + presencePenalty: Optional[dict] = None + countPenalty: Optional[dict] = None + + def __init__( + self, + maxTokens: Optional[int] = None, + temperature: Optional[float] = None, + topP: Optional[float] = None, + stopSequences: Optional[list] = None, + frequencePenalty: Optional[dict] = None, + presencePenalty: Optional[dict] = None, + countPenalty: Optional[dict] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + AmazonInvokeConfig.__init__(self) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self, model: str) -> List: + return [ + "max_tokens", + "temperature", + "top_p", + "stream", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for k, v in non_default_params.items(): + if k == "max_tokens": + optional_params["maxTokens"] = v + if k == "temperature": + optional_params["temperature"] = v + if k == "top_p": + optional_params["topP"] = v + if k == "stream": + optional_params["stream"] = v + return optional_params diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..9c2c95e6cea9ac11b741ddd197626fcf7d7f9711 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py @@ -0,0 +1,75 @@ +import types +from typing import List, Optional + +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) +from litellm.llms.cohere.chat.transformation import CohereChatConfig + + +class AmazonCohereConfig(AmazonInvokeConfig, CohereChatConfig): + """ + Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=command + + Supported Params for the Amazon / Cohere models: + + - `max_tokens` (integer) max tokens, + - `temperature` (float) model temperature, + - `return_likelihood` (string) n/a + """ + + max_tokens: Optional[int] = None + return_likelihood: Optional[str] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + return_likelihood: Optional[str] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + AmazonInvokeConfig.__init__(self) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self, model: str) -> List[str]: + supported_params = CohereChatConfig.get_supported_openai_params( + self, model=model + ) + return supported_params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + return CohereChatConfig.map_openai_params( + self, + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..0fe84b0ce0c36da3bc8431747224e2ce26626116 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -0,0 +1,135 @@ +from typing import Any, List, Optional, cast + +from httpx import Response + +from litellm import verbose_logger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning, +) +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + LiteLLMLoggingObj, +) +from litellm.types.llms.bedrock import AmazonDeepSeekR1StreamingResponse +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ( + ChatCompletionUsageBlock, + Choices, + Delta, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) + +from .amazon_llama_transformation import AmazonLlamaConfig + + +class AmazonDeepSeekR1Config(AmazonLlamaConfig): + def transform_response( + self, + model: str, + raw_response: Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Extract the reasoning content, and return it as a separate field in the response. + """ + response = super().transform_response( + model, + raw_response, + model_response, + logging_obj, + request_data, + messages, + optional_params, + litellm_params, + encoding, + api_key, + json_mode, + ) + prompt = cast(Optional[str], request_data.get("prompt")) + message_content = cast( + Optional[str], cast(Choices, response.choices[0]).message.get("content") + ) + if prompt and prompt.strip().endswith("") and message_content: + message_content_with_reasoning_token = "" + message_content + reasoning, content = _parse_content_for_reasoning( + message_content_with_reasoning_token + ) + provider_specific_fields = ( + cast(Choices, response.choices[0]).message.provider_specific_fields + or {} + ) + if reasoning: + provider_specific_fields["reasoning_content"] = reasoning + + message = Message( + **{ + **cast(Choices, response.choices[0]).message.model_dump(), + "content": content, + "provider_specific_fields": provider_specific_fields, + } + ) + cast(Choices, response.choices[0]).message = message + return response + + +class AmazonDeepseekR1ResponseIterator(BaseModelResponseIterator): + def __init__(self, streaming_response: Any, sync_stream: bool) -> None: + super().__init__(streaming_response=streaming_response, sync_stream=sync_stream) + self.has_finished_thinking = False + + def chunk_parser(self, chunk: dict) -> ModelResponseStream: + """ + Deepseek r1 starts by thinking, then it generates the response. + """ + try: + typed_chunk = AmazonDeepSeekR1StreamingResponse(**chunk) # type: ignore + generated_content = typed_chunk["generation"] + if generated_content == "" and not self.has_finished_thinking: + verbose_logger.debug( + "Deepseek r1: received, setting has_finished_thinking to True" + ) + generated_content = "" + self.has_finished_thinking = True + + prompt_token_count = typed_chunk.get("prompt_token_count") or 0 + generation_token_count = typed_chunk.get("generation_token_count") or 0 + usage = ChatCompletionUsageBlock( + prompt_tokens=prompt_token_count, + completion_tokens=generation_token_count, + total_tokens=prompt_token_count + generation_token_count, + ) + + return ModelResponseStream( + choices=[ + StreamingChoices( + finish_reason=typed_chunk["stop_reason"], + delta=Delta( + content=( + generated_content + if self.has_finished_thinking + else None + ), + reasoning_content=( + generated_content + if not self.has_finished_thinking + else None + ), + ), + ) + ], + usage=usage, + ) + + except Exception as e: + raise e diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..9f84844fcb6aa1486db2aede988ad8bcaebf8f16 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py @@ -0,0 +1,80 @@ +import types +from typing import List, Optional + +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) + + +class AmazonLlamaConfig(AmazonInvokeConfig, BaseConfig): + """ + Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=meta.llama2-13b-chat-v1 + + Supported Params for the Amazon / Meta Llama models: + + - `max_gen_len` (integer) max tokens, + - `temperature` (float) temperature for model, + - `top_p` (float) top p for model + """ + + max_gen_len: Optional[int] = None + temperature: Optional[float] = None + topP: Optional[float] = None + + def __init__( + self, + maxTokenCount: Optional[int] = None, + temperature: Optional[float] = None, + topP: Optional[int] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + AmazonInvokeConfig.__init__(self) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self, model: str) -> List: + return [ + "max_tokens", + "temperature", + "top_p", + "stream", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for k, v in non_default_params.items(): + if k == "max_tokens": + optional_params["max_gen_len"] = v + if k == "temperature": + optional_params["temperature"] = v + if k == "top_p": + optional_params["top_p"] = v + if k == "stream": + optional_params["stream"] = v + return optional_params diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..3992de4d4fc487e078f13442ab23678c83793f5a --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py @@ -0,0 +1,119 @@ +import types +from typing import List, Optional, TYPE_CHECKING + +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) +from litellm.llms.bedrock.common_utils import BedrockError + +if TYPE_CHECKING: + from litellm.types.utils import ModelResponse + + +class AmazonMistralConfig(AmazonInvokeConfig, BaseConfig): + """ + Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mistral.html + Supported Params for the Amazon / Mistral models: + + - `max_tokens` (integer) max tokens, + - `temperature` (float) temperature for model, + - `top_p` (float) top p for model + - `stop` [string] A list of stop sequences that if generated by the model, stops the model from generating further output. + - `top_k` (float) top k for model + """ + + max_tokens: Optional[int] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[float] = None + stop: Optional[List[str]] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + top_p: Optional[int] = None, + top_k: Optional[float] = None, + stop: Optional[List[str]] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + AmazonInvokeConfig.__init__(self) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self, model: str) -> List[str]: + return ["max_tokens", "temperature", "top_p", "stop", "stream"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for k, v in non_default_params.items(): + if k == "max_tokens": + optional_params["max_tokens"] = v + if k == "temperature": + optional_params["temperature"] = v + if k == "top_p": + optional_params["top_p"] = v + if k == "stop": + optional_params["stop"] = v + if k == "stream": + optional_params["stream"] = v + return optional_params + + @staticmethod + def get_outputText( + completion_response: dict, model_response: "ModelResponse" + ) -> str: + """This function extracts the output text from a bedrock mistral completion. + As a side effect, it updates the finish reason for a model response. + + Args: + completion_response: JSON from the completion. + model_response: ModelResponse + + Returns: + A string with the response of the LLM + + """ + if "choices" in completion_response: + outputText = completion_response["choices"][0]["message"]["content"] + model_response.choices[0].finish_reason = completion_response["choices"][0][ + "finish_reason" + ] + elif "outputs" in completion_response: + outputText = completion_response["outputs"][0]["text"] + model_response.choices[0].finish_reason = completion_response["outputs"][0][ + "stop_reason" + ] + else: + raise BedrockError( + message="Unexpected mistral completion response", status_code=400 + ) + + return outputText diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..3aeb65b58c709b721b628b5ad7f81e042bbbebbc --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -0,0 +1,266 @@ +""" +Transformation for Bedrock Moonshot AI (Kimi K2) models. + +Supports the Kimi K2 Thinking model available on Amazon Bedrock. +Model format: bedrock/moonshot.kimi-k2-thinking-v1:0 + +Reference: https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/ +""" + +from typing import TYPE_CHECKING, Any, List, Optional, Union +import re + +import httpx + +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.types.utils import ModelResponse + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): + """ + Configuration for Bedrock Moonshot AI (Kimi K2) models. + + Reference: + https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/ + https://platform.moonshot.ai/docs/api/chat + + Supported Params for the Amazon / Moonshot models: + - `max_tokens` (integer) max tokens + - `temperature` (float) temperature for model (0-1 for Moonshot) + - `top_p` (float) top p for model + - `stream` (bool) whether to stream responses + - `tools` (list) tool definitions (supported on kimi-k2-thinking) + - `tool_choice` (str|dict) tool choice specification (supported on kimi-k2-thinking) + + NOT Supported on Bedrock: + - `stop` sequences (Bedrock doesn't support stopSequences field for this model) + + Note: The kimi-k2-thinking model DOES support tool calls, unlike kimi-thinking-preview. + """ + + def __init__(self, **kwargs): + AmazonInvokeConfig.__init__(self, **kwargs) + MoonshotChatConfig.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + + def _get_model_id(self, model: str) -> str: + """ + Extract the actual model ID from the LiteLLM model name. + + Removes routing prefixes like: + - bedrock/invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking + - invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking + - moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking + """ + # Remove bedrock/ prefix if present + if model.startswith("bedrock/"): + model = model[8:] + + # Remove invoke/ prefix if present + if model.startswith("invoke/"): + model = model[7:] + + # Remove any provider prefix (e.g., moonshot/) + if "/" in model and not model.startswith("arn:"): + parts = model.split("/", 1) + if len(parts) == 2: + model = parts[1] + + return model + + def get_supported_openai_params(self, model: str) -> List[str]: + """ + Get the supported OpenAI params for Moonshot AI models on Bedrock. + + Bedrock-specific limitations: + - stopSequences field is not supported on Bedrock (unlike native Moonshot API) + - functions parameter is not supported (use tools instead) + - tool_choice doesn't support "required" value + + Note: kimi-k2-thinking DOES support tool calls (unlike kimi-thinking-preview) + The parent MoonshotChatConfig class handles the kimi-thinking-preview exclusion. + """ + excluded_params: List[str] = [ + "functions", + "stop", + ] # Bedrock doesn't support stopSequences + + base_openai_params = super( + MoonshotChatConfig, self + ).get_supported_openai_params(model=model) + final_params: List[str] = [] + for param in base_openai_params: + if param not in excluded_params: + final_params.append(param) + + return final_params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Moonshot AI parameters for Bedrock. + + Handles Moonshot AI specific limitations: + - tool_choice doesn't support "required" value + - Temperature <0.3 limitation for n>1 + - Temperature range is [0, 1] (not [0, 2] like OpenAI) + """ + return MoonshotChatConfig.map_openai_params( + self, + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request for Bedrock Moonshot AI models. + + Uses the Moonshot transformation logic which handles: + - Converting content lists to strings (Moonshot doesn't support list format) + - Adding tool_choice="required" message if needed + - Temperature and parameter validation + + """ + # Filter out AWS credentials using the existing method from BaseAWSLLM + self._get_boto_credentials_from_optional_params(optional_params, model) + + # Strip routing prefixes to get the actual model ID + clean_model_id = self._get_model_id(model) + + # Use Moonshot's transform_request which handles message transformation + # and tool_choice="required" workaround + return MoonshotChatConfig.transform_request( + self, + model=clean_model_id, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + def _extract_reasoning_from_content( + self, content: str + ) -> tuple[Optional[str], str]: + """ + Extract reasoning content from tags in the response. + + Moonshot AI's Kimi K2 Thinking model returns reasoning in tags. + This method extracts that content and returns it separately. + + Args: + content: The full content string from the API response + + Returns: + tuple: (reasoning_content, main_content) + """ + if not content: + return None, content + + # Match ... tags + reasoning_match = re.match( + r"(.*?)\s*(.*)", content, re.DOTALL + ) + + if reasoning_match: + reasoning_content = reasoning_match.group(1).strip() + main_content = reasoning_match.group(2).strip() + return reasoning_content, main_content + + return None, content + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: "ModelResponse", + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> "ModelResponse": + """ + Transform the response from Bedrock Moonshot AI models. + + Moonshot AI uses OpenAI-compatible response format, but returns reasoning + content in tags. This method: + 1. Calls parent class transformation + 2. Extracts reasoning content from tags + 3. Sets reasoning_content on the message object + """ + # First, get the standard transformation + model_response = MoonshotChatConfig.transform_response( + self, + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + + # Extract reasoning content from tags + if model_response.choices and len(model_response.choices) > 0: + for choice in model_response.choices: + # Only process Choices (not StreamingChoices) which have message attribute + if ( + isinstance(choice, Choices) + and choice.message + and choice.message.content + ): + ( + reasoning_content, + main_content, + ) = self._extract_reasoning_from_content(choice.message.content) + + if reasoning_content: + # Set the reasoning_content field + choice.message.reasoning_content = reasoning_content + # Update the main content without reasoning tags + choice.message.content = main_content + + return model_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BedrockError: + """Return the appropriate error class for Bedrock.""" + return BedrockError(status_code=status_code, message=error_message) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..3506c8f1cc0750dee6bf89343c127f714c8f98a1 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -0,0 +1,120 @@ +""" +Handles transforming requests for `bedrock/invoke/{nova} models` + +Inherits from `AmazonConverseConfig` + +Nova + Invoke API Tutorial: https://docs.aws.amazon.com/nova/latest/userguide/using-invoke-api.html +""" + +from typing import Any, List, Optional + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.types.llms.bedrock import BedrockInvokeNovaRequest +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse + +from ..converse_transformation import AmazonConverseConfig +from .base_invoke_transformation import AmazonInvokeConfig + + +class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): + """ + Config for sending `nova` requests to `/bedrock/invoke/` + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_supported_openai_params(self, model: str) -> list: + return AmazonConverseConfig.get_supported_openai_params(self, model) + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + return AmazonConverseConfig.map_openai_params( + self, non_default_params, optional_params, model, drop_params + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + _transformed_nova_request = AmazonConverseConfig.transform_request( + self, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + _bedrock_invoke_nova_request = BedrockInvokeNovaRequest( + **_transformed_nova_request + ) + self._remove_empty_system_messages(_bedrock_invoke_nova_request) + bedrock_invoke_nova_request = self._filter_allowed_fields( + _bedrock_invoke_nova_request + ) + return bedrock_invoke_nova_request + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: Logging, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + return AmazonConverseConfig.transform_response( + self, + model, + raw_response, + model_response, + logging_obj, + request_data, + messages, + optional_params, + litellm_params, + encoding, + api_key, + json_mode, + ) + + def _filter_allowed_fields( + self, bedrock_invoke_nova_request: BedrockInvokeNovaRequest + ) -> dict: + """ + Filter out fields that are not allowed in the `BedrockInvokeNovaRequest` dataclass. + """ + allowed_fields = set(BedrockInvokeNovaRequest.__annotations__.keys()) + return { + k: v for k, v in bedrock_invoke_nova_request.items() if k in allowed_fields + } + + def _remove_empty_system_messages( + self, bedrock_invoke_nova_request: BedrockInvokeNovaRequest + ) -> None: + """ + In-place remove empty `system` messages from the request. + + /bedrock/invoke/ does not allow empty `system` messages. + """ + _system_message = bedrock_invoke_nova_request.get("system", None) + if isinstance(_system_message, list) and len(_system_message) == 0: + bedrock_invoke_nova_request.pop("system", None) + return diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..7b64c6066d00d69e6d518d23694aa78d386e9e4d --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py @@ -0,0 +1,192 @@ +""" +Transformation for Bedrock imported models that use OpenAI Chat Completions format. + +Use this for models imported into Bedrock that accept the OpenAI API format. +Model format: bedrock/openai/ + +Example: bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123 +""" + +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union + +import httpx + +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.passthrough.utils import CommonUtils +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM): + """ + Configuration for Bedrock imported models that use OpenAI Chat Completions format. + + This class handles the transformation of requests and responses for Bedrock + imported models that accept the OpenAI API format directly. + + Inherits from OpenAIGPTConfig to leverage standard OpenAI parameter handling + and response transformation, while adding Bedrock-specific URL generation + and AWS request signing. + + Usage: + model = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123" + """ + + def __init__(self, **kwargs): + OpenAIGPTConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + + def _get_openai_model_id(self, model: str) -> str: + """ + Extract the actual model ID from the LiteLLM model name. + + Input format: bedrock/openai/ + Returns: + """ + # Remove bedrock/ prefix if present + if model.startswith("bedrock/"): + model = model[8:] + + # Remove openai/ prefix + if model.startswith("openai/"): + model = model[7:] + + return model + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the Bedrock invoke endpoint. + + Uses the standard Bedrock invoke endpoint format. + """ + model_id = self._get_openai_model_id(model) + + # Get AWS region + aws_region_name = self._get_aws_region_name( + optional_params=optional_params, model=model + ) + + # Get runtime endpoint + aws_bedrock_runtime_endpoint = optional_params.get( + "aws_bedrock_runtime_endpoint", None + ) + endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_region_name=aws_region_name, + ) + + # Encode model ID for ARNs (e.g., :imported-model/ -> :imported-model%2F) + model_id = CommonUtils.encode_bedrock_runtime_modelid_arn(model_id) + + # Build the invoke URL + if stream: + endpoint_url = ( + f"{endpoint_url}/model/{model_id}/invoke-with-response-stream" + ) + else: + endpoint_url = f"{endpoint_url}/model/{model_id}/invoke" + + return endpoint_url + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + """ + Sign the request using AWS Signature Version 4. + """ + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to OpenAI Chat Completions format for Bedrock imported models. + + Removes AWS-specific params and stream param (handled separately in URL), + then delegates to parent class for standard OpenAI request transformation. + """ + # Remove stream from optional_params as it's handled separately in URL + optional_params.pop("stream", None) + + # Remove AWS-specific params that shouldn't be in the request body + inference_params = { + k: v + for k, v in optional_params.items() + if k not in self.aws_authentication_params + } + + # Use parent class transform_request for OpenAI format + return super().transform_request( + model=self._get_openai_model_id(model), + messages=messages, + optional_params=inference_params, + litellm_params=litellm_params, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate the environment and return headers. + + For Bedrock, we don't need Bearer token auth since we use AWS SigV4. + """ + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BedrockError: + """Return the appropriate error class for Bedrock.""" + return BedrockError(status_code=status_code, message=error_message) diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..c65e9e0b083244fb6f9c3ff58e2e0e396b509098 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -0,0 +1,99 @@ +""" +Handles transforming requests for `bedrock/invoke/{qwen2} models` + +Inherits from `AmazonQwen3Config` since Qwen2 and Qwen3 architectures are mostly similar. +The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field. + +Qwen2 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html +""" + +from typing import Any, List, Optional + +import httpx + +from litellm.llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( + AmazonQwen3Config, +) +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + LiteLLMLoggingObj, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse, Usage + + +class AmazonQwen2Config(AmazonQwen3Config): + """ + Config for sending `qwen2` requests to `/bedrock/invoke/` + + Inherits from AmazonQwen3Config since Qwen2 and Qwen3 architectures are mostly similar. + The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field. + + Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html + """ + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform Qwen2 Bedrock response to OpenAI format + + Qwen2 uses "text" field, but we also support "generation" field for compatibility. + """ + try: + if hasattr(raw_response, "json"): + response_data = raw_response.json() + else: + response_data = raw_response + + # Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility + generated_text = response_data.get("generation", "") or response_data.get( + "text", "" + ) + + # Clean up the response (remove assistant start token if present) + if generated_text.startswith("<|im_start|>assistant\n"): + generated_text = generated_text[len("<|im_start|>assistant\n") :] + if generated_text.endswith("<|im_end|>"): + generated_text = generated_text[: -len("<|im_end|>")] + + # Set the content in the existing model_response structure + if hasattr(model_response, "choices") and len(model_response.choices) > 0: + choice = model_response.choices[0] + choice.message.content = generated_text + choice.finish_reason = "stop" + + # Set usage information if available in response + if "usage" in response_data: + usage_data = response_data["usage"] + setattr( + model_response, + "usage", + Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + ), + ) + + return model_response + + except Exception as e: + if logging_obj: + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=raw_response, + additional_args={"error": str(e)}, + ) + raise e diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..6325c388181c0c37abbee1a349c732aca93e4bdb --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -0,0 +1,225 @@ +""" +Handles transforming requests for `bedrock/invoke/{qwen3} models` + +Inherits from `AmazonInvokeConfig` + +Qwen3 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html +""" + +from typing import Any, List, Optional + +import httpx + +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, + LiteLLMLoggingObj, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse, Usage + + +class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): + """ + Config for sending `qwen3` requests to `/bedrock/invoke/` + + Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html + """ + + max_tokens: Optional[int] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[int] = None + stop: Optional[List[str]] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + stop: Optional[List[str]] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + AmazonInvokeConfig.__init__(self) + + def get_supported_openai_params(self, model: str) -> List[str]: + return [ + "max_tokens", + "temperature", + "top_p", + "top_k", + "stop", + "stream", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for k, v in non_default_params.items(): + if k == "max_tokens": + optional_params["max_tokens"] = v + if k == "temperature": + optional_params["temperature"] = v + if k == "top_p": + optional_params["top_p"] = v + if k == "top_k": + optional_params["top_k"] = v + if k == "stop": + optional_params["stop"] = v + if k == "stream": + optional_params["stream"] = v + return optional_params + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform OpenAI format to Qwen3 Bedrock invoke format + """ + # Convert messages to prompt format + prompt = self._convert_messages_to_prompt(messages) + + # Build the request body + request_body = { + "prompt": prompt, + } + + # Add optional parameters + if "max_tokens" in optional_params: + request_body["max_gen_len"] = optional_params["max_tokens"] + if "temperature" in optional_params: + request_body["temperature"] = optional_params["temperature"] + if "top_p" in optional_params: + request_body["top_p"] = optional_params["top_p"] + if "top_k" in optional_params: + request_body["top_k"] = optional_params["top_k"] + if "stop" in optional_params: + request_body["stop"] = optional_params["stop"] + + return request_body + + def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str: + """ + Convert OpenAI messages format to Qwen3 prompt format + Supports tool calls, multimodal content, and various message types + """ + prompt_parts = [] + + for message in messages: + role = message.get("role", "") + content = message.get("content", "") + tool_calls = message.get("tool_calls", []) + + if role == "system": + prompt_parts.append(f"<|im_start|>system\n{content}<|im_end|>") + elif role == "user": + # Handle multimodal content + if isinstance(content, list): + text_content = [] + for item in content: + if item.get("type") == "text": + text_content.append(item.get("text", "")) + elif item.get("type") == "image_url": + # For Qwen3, we can include image placeholders + text_content.append( + "<|vision_start|><|image_pad|><|vision_end|>" + ) + content = "".join(text_content) + prompt_parts.append(f"<|im_start|>user\n{content}<|im_end|>") + elif role == "assistant": + if tool_calls and isinstance(tool_calls, list): + # Handle tool calls + for tool_call in tool_calls: + function_name = tool_call.get("function", {}).get("name", "") + function_args = tool_call.get("function", {}).get( + "arguments", "" + ) + prompt_parts.append( + f'<|im_start|>assistant\n\n{{"name": "{function_name}", "arguments": "{function_args}"}}\n<|im_end|>' + ) + else: + prompt_parts.append(f"<|im_start|>assistant\n{content}<|im_end|>") + elif role == "tool": + # Handle tool responses + prompt_parts.append(f"<|im_start|>tool\n{content}<|im_end|>") + + # Add assistant start token for response generation + prompt_parts.append("<|im_start|>assistant\n") + + return "\n".join(prompt_parts) + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform Qwen3 Bedrock response to OpenAI format + """ + try: + if hasattr(raw_response, "json"): + response_data = raw_response.json() + else: + response_data = raw_response + + # Extract the generated text - Qwen3 uses "generation" field + generated_text = response_data.get("generation", "") + + # Clean up the response (remove assistant start token if present) + if generated_text.startswith("<|im_start|>assistant\n"): + generated_text = generated_text[len("<|im_start|>assistant\n") :] + if generated_text.endswith("<|im_end|>"): + generated_text = generated_text[: -len("<|im_end|>")] + + # Set the content in the existing model_response structure + if hasattr(model_response, "choices") and len(model_response.choices) > 0: + choice = model_response.choices[0] + choice.message.content = generated_text + choice.finish_reason = "stop" + + # Set usage information if available in response + if "usage" in response_data: + usage_data = response_data["usage"] + setattr( + model_response, + "usage", + Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + ), + ) + + return model_response + + except Exception as e: + if logging_obj: + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=raw_response, + additional_args={"error": str(e)}, + ) + raise e diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..367fb84d1ac6dc52058630d149897c964ff00851 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py @@ -0,0 +1,116 @@ +import re +import types +from typing import List, Optional, Union + +import litellm +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) + + +class AmazonTitanConfig(AmazonInvokeConfig, BaseConfig): + """ + Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=titan-text-express-v1 + + Supported Params for the Amazon Titan models: + + - `maxTokenCount` (integer) max tokens, + - `stopSequences` (string[]) list of stop sequence strings + - `temperature` (float) temperature for model, + - `topP` (int) top p for model + """ + + maxTokenCount: Optional[int] = None + stopSequences: Optional[list] = None + temperature: Optional[float] = None + topP: Optional[int] = None + + def __init__( + self, + maxTokenCount: Optional[int] = None, + stopSequences: Optional[list] = None, + temperature: Optional[float] = None, + topP: Optional[int] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + AmazonInvokeConfig.__init__(self) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def _map_and_modify_arg( + self, + supported_params: dict, + provider: str, + model: str, + stop: Union[List[str], str], + ): + """ + filter params to fit the required provider format, drop those that don't fit if user sets `litellm.drop_params = True`. + """ + filtered_stop = None + if "stop" in supported_params and litellm.drop_params: + if provider == "bedrock" and "amazon" in model: + filtered_stop = [] + if isinstance(stop, list): + for s in stop: + if re.match(r"^(\|+|User:)$", s): + filtered_stop.append(s) + if filtered_stop is not None: + supported_params["stop"] = filtered_stop + + return supported_params + + def get_supported_openai_params(self, model: str) -> List[str]: + return [ + "max_tokens", + "max_completion_tokens", + "stop", + "temperature", + "top_p", + "stream", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for k, v in non_default_params.items(): + if k == "max_tokens" or k == "max_completion_tokens": + optional_params["maxTokenCount"] = v + if k == "temperature": + optional_params["temperature"] = v + if k == "stop": + filtered_stop = self._map_and_modify_arg( + {"stop": v}, provider="bedrock", model=model, stop=v + ) + optional_params["stopSequences"] = filtered_stop["stop"] + if k == "top_p": + optional_params["topP"] = v + if k == "stream": + optional_params["stream"] = v + return optional_params diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..889480d31a57b1e551f8cdd8ea551ca60463bca8 --- /dev/null +++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -0,0 +1,280 @@ +""" +Transforms OpenAI-style requests into TwelveLabs Pegasus 1.2 requests for Bedrock. + +Reference: +https://docs.twelvelabs.io/docs/models/pegasus +""" + +import json +import time +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.llms.base_llm.base_utils import type_to_response_format_param +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse, Usage +from litellm.utils import get_base64_str + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): + """ + Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for + `twelvelabs.pegasus-1-2-v1:0`. + + Pegasus 1.2 requires an `inputPrompt` and a `mediaSource` that either references + an S3 object or a base64-encoded clip. Optional OpenAI params (temperature, + response_format, max_tokens) are translated to the TwelveLabs schema. + """ + + def get_supported_openai_params(self, model: str) -> List[str]: + return [ + "max_tokens", + "max_completion_tokens", + "temperature", + "response_format", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for param, value in non_default_params.items(): + if param in {"max_tokens", "max_completion_tokens"}: + optional_params["maxOutputTokens"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "response_format": + optional_params["responseFormat"] = self._normalize_response_format( + value + ) + return optional_params + + def _normalize_response_format(self, value: Any) -> Any: + """Normalize response_format to TwelveLabs format. + + TwelveLabs expects: + { + "jsonSchema": {...} + } + + But OpenAI format is: + { + "type": "json_schema", + "json_schema": { + "name": "...", + "schema": {...} + } + } + """ + if isinstance(value, dict): + # If it has json_schema field, extract and transform it + if "json_schema" in value: + json_schema = value["json_schema"] + # Extract the schema if nested + if isinstance(json_schema, dict) and "schema" in json_schema: + return {"jsonSchema": json_schema["schema"]} + # Otherwise use json_schema directly + return {"jsonSchema": json_schema} + # If it already has jsonSchema, return as is + if "jsonSchema" in value: + return value + # Otherwise return the dict as is + return value + return type_to_response_format_param(response_format=value) or value + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + input_prompt = self._convert_messages_to_prompt(messages=messages) + request_data: Dict[str, Any] = {"inputPrompt": input_prompt} + + media_source = self._build_media_source(optional_params) + if media_source is not None: + request_data["mediaSource"] = media_source + + # Handle temperature and maxOutputTokens + for key in ("temperature", "maxOutputTokens"): + if key in optional_params: + request_data[key] = optional_params.get(key) + + # Handle responseFormat - transform to TwelveLabs format + if "responseFormat" in optional_params: + response_format = optional_params["responseFormat"] + transformed_format = self._normalize_response_format(response_format) + if transformed_format: + request_data["responseFormat"] = transformed_format + + return request_data + + def _build_media_source(self, optional_params: dict) -> Optional[dict]: + direct_source = optional_params.get("mediaSource") or optional_params.get( + "media_source" + ) + if isinstance(direct_source, dict): + return direct_source + + base64_input = optional_params.get("video_base64") or optional_params.get( + "base64_string" + ) + if base64_input: + return {"base64String": get_base64_str(base64_input)} + + s3_uri = ( + optional_params.get("video_s3_uri") + or optional_params.get("s3_uri") + or optional_params.get("media_source_s3_uri") + ) + if s3_uri: + s3_location = {"uri": s3_uri} + bucket_owner = ( + optional_params.get("video_s3_bucket_owner") + or optional_params.get("s3_bucket_owner") + or optional_params.get("media_source_bucket_owner") + ) + if bucket_owner: + s3_location["bucketOwner"] = bucket_owner + return {"s3Location": s3_location} + return None + + def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str: + prompt_parts: List[str] = [] + for message in messages: + role = message.get("role", "user") + content = message.get("content", "") + if isinstance(content, list): + text_fragments = [] + for item in content: + if isinstance(item, dict): + item_type = item.get("type") + if item_type == "text": + text_fragments.append(item.get("text", "")) + elif item_type == "image_url": + text_fragments.append("") + elif item_type == "video_url": + text_fragments.append("