feat(gpt): add GPT-5.4 support

embodied_gen/utils/gpt_clients.py

@@ -14,18 +14,21 @@
 # implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-import math
+
 import base64
 import logging
+import math
 import os
 from io import BytesIO
 from typing import Optional
 
+import openai
 import yaml
 from openai import AzureOpenAI, OpenAI  # pip install openai
 from PIL import Image
 from tenacity import (
     retry,
+    retry_if_not_exception_type,
     stop_after_attempt,
     stop_after_delay,
     wait_random_exponential,
@@ -40,7 +43,12 @@ __all__ = [
     "GPTclient",
 ]
 
-CONFIG_FILE = "embodied_gen/utils/gpt_config.yaml"
+_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+CONFIG_FILE = os.path.join(_CURRENT_DIR, "gpt_config.yaml")
+DEFAULT_GPT_TIMEOUT = float(os.environ.get("GPT_TIMEOUT", 120))
+# GPT-5.x counts reasoning tokens against this cap, so it must be high
+# enough to leave room for both reasoning and the visible reply.
+GPT5_DEFAULT_MAX_COMPLETION_TOKENS = 8192
 
 
 def combine_images_to_grid(
@@ -76,7 +84,36 @@ def combine_images_to_grid(
 
 
 class GPTclient:
-    """A client to interact with the GPT model via OpenAI or Azure API."""
+    """A client to interact with GPT models via OpenAI or Azure API.
+
+    Supports text and image prompts, connection checking, and configurable parameters.
+
+    Args:
+        endpoint (str): API endpoint URL.
+        api_key (str): API key for authentication.
+        model_name (str, optional): Model name to use.
+        api_version (str, optional): API version (for Azure).
+        check_connection (bool, optional): Whether to check API connection.
+        verbose (bool, optional): Enable verbose logging.
+        timeout (float, optional): Max seconds for a single GPT request.
+
+    Example:
+        ```sh
+        export ENDPOINT="https://yfb-openai-sweden.openai.azure.com"
+        export API_KEY="xxxxxx"
+        export API_VERSION="2025-03-01-preview"
+        export MODEL_NAME="yfb-gpt-4o-sweden"
+        ```
+        ```py
+        from embodied_gen.utils.gpt_clients import GPT_CLIENT
+
+        response = GPT_CLIENT.query("Describe the physics of a falling apple.")
+        response = GPT_CLIENT.query(
+            text_prompt="Describe the content in each image."
+            image_base64=["path/to/image1.png", "path/to/image2.jpg"],
+        )
+        ```
+    """
 
     def __init__(
         self,
@@ -86,21 +123,27 @@ class GPTclient:
         api_version: str = None,
         check_connection: bool = True,
         verbose: bool = False,
+        timeout: float = DEFAULT_GPT_TIMEOUT,
     ):
         if api_version is not None:
             self.client = AzureOpenAI(
                 azure_endpoint=endpoint,
                 api_key=api_key,
                 api_version=api_version,
+                timeout=timeout,
+                max_retries=0,
             )
         else:
             self.client = OpenAI(
                 base_url=endpoint,
                 api_key=api_key,
+                timeout=timeout,
+                max_retries=0,
             )
 
         self.endpoint = endpoint
         self.model_name = model_name
+        self.timeout = timeout
         self.image_formats = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif"}
         self.verbose = verbose
         if check_connection:
@@ -108,11 +151,18 @@ class GPTclient:
 
         logger.info(f"Using GPT model: {self.model_name}.")
 
+    @staticmethod
+    def _is_gpt5_model(model_name: str) -> bool:
+        name = (model_name or "").lower()
+        return "gpt-5" in name or "gpt5" in name
+
     @retry(
-        wait=wait_random_exponential(min=1, max=20),
-        stop=(stop_after_attempt(10) | stop_after_delay(30)),
+        retry=retry_if_not_exception_type(openai.BadRequestError),
+        wait=wait_random_exponential(min=1, max=10),
+        stop=stop_after_attempt(5) | stop_after_delay(DEFAULT_GPT_TIMEOUT),
     )
     def completion_with_backoff(self, **kwargs):
+        """Performs a chat completion request with retry/backoff."""
         return self.client.chat.completions.create(**kwargs)
 
     def query(
@@ -122,19 +172,16 @@ class GPTclient:
         system_role: Optional[str] = None,
         params: Optional[dict] = None,
     ) -> Optional[str]:
-        """Queries the GPT model with a text and optional image prompts.
+        """Queries the GPT model with text and optional image prompts.
 
         Args:
-            text_prompt (str): The main text input that the model responds to.
-            image_base64 (Optional[List[str]]): A list of image base64 strings
-                or local image paths or PIL.Image to accompany the text prompt.
-            system_role (Optional[str]): Optional system-level instructions
-                that specify the behavior of the assistant.
-            params (Optional[dict]): Additional parameters for GPT setting.
+            text_prompt (str): Main text input.
+            image_base64 (Optional[list[str | Image.Image]], optional): List of image base64 strings, file paths, or PIL Images.
+            system_role (Optional[str], optional): System-level instructions.
+            params (Optional[dict], optional): Additional GPT parameters.
 
         Returns:
-            Optional[str]: The response content generated by the model based on
-                the prompt. Returns `None` if an error occurs.
+            Optional[str]: Model response content, or None if error.
         """
         if system_role is None:
             system_role = "You are a highly knowledgeable assistant specializing in physics, engineering, and object properties."  # noqa
@@ -176,21 +223,49 @@ class GPTclient:
                     }
                 )
 
-        payload = {
-            "messages": [
-                {"role": "system", "content": system_role},
-                {"role": "user", "content": content_user},
-            ],
-            "temperature": 0.1,
-            "max_tokens": 500,
-            "top_p": 0.1,
-            "frequency_penalty": 0,
-            "presence_penalty": 0,
-            "stop": None,
-            "model": self.model_name,
-        }
+        is_gpt5 = self._is_gpt5_model(self.model_name)
+        if is_gpt5:
+            # GPT-5.x only supports default temperature/top_p and uses
+            # `max_completion_tokens` instead of `max_tokens`.
+            payload = {
+                "messages": [
+                    {"role": "system", "content": system_role},
+                    {"role": "user", "content": content_user},
+                ],
+                "max_completion_tokens": GPT5_DEFAULT_MAX_COMPLETION_TOKENS,
+                "model": self.model_name,
+            }
+        else:
+            payload = {
+                "messages": [
+                    {"role": "system", "content": system_role},
+                    {"role": "user", "content": content_user},
+                ],
+                "temperature": 0.1,
+                "max_tokens": 500,
+                "top_p": 0.1,
+                "frequency_penalty": 0,
+                "presence_penalty": 0,
+                "stop": None,
+                "model": self.model_name,
+            }
 
         if params:
+            params = dict(params)
+            if is_gpt5:
+                # GPT-5.x rejects custom temperature/top_p/penalty/stop and
+                # uses `max_completion_tokens` instead of `max_tokens`.
+                if "max_tokens" in params and "max_completion_tokens" not in params:
+                    params["max_completion_tokens"] = params.pop("max_tokens")
+                for k in (
+                    "temperature",
+                    "top_p",
+                    "frequency_penalty",
+                    "presence_penalty",
+                    "stop",
+                    "max_tokens",
+                ):
+                    params.pop(k, None)
             payload.update(params)
 
         response = None
@@ -208,20 +283,28 @@ class GPTclient:
         return response
 
     def check_connection(self) -> None:
-        """Check whether the GPT API connection is working."""
+        """Checks whether the GPT API connection is working.
+
+        Raises:
+            ConnectionError: If connection fails.
+        """
         try:
-            response = self.completion_with_backoff(
+            probe_kwargs = dict(
                 messages=[
                     {"role": "system", "content": "You are a test system."},
                     {"role": "user", "content": "Hello"},
                 ],
                 model=self.model_name,
-                temperature=0,
-                max_tokens=100,
             )
-            content = response.choices[0].message.content
-            logger.info(f"Connection check success.")
-        except Exception as e:
+            if self._is_gpt5_model(self.model_name):
+                probe_kwargs["max_completion_tokens"] = 100
+            else:
+                probe_kwargs["temperature"] = 0
+                probe_kwargs["max_tokens"] = 100
+            response = self.completion_with_backoff(**probe_kwargs)
+            response.choices[0].message.content
+            logger.info("Connection check success.")
+        except Exception:
             raise ConnectionError(
                 f"Failed to connect to GPT API at {self.endpoint}, "
                 f"please check setting in `{CONFIG_FILE}` and `README`."
@@ -239,6 +322,7 @@ endpoint = os.environ.get("ENDPOINT", agent_config.get("endpoint"))
 api_key = os.environ.get("API_KEY", agent_config.get("api_key"))
 api_version = os.environ.get("API_VERSION", agent_config.get("api_version"))
 model_name = os.environ.get("MODEL_NAME", agent_config.get("model_name"))
+timeout = DEFAULT_GPT_TIMEOUT
 
 GPT_CLIENT = GPTclient(
     endpoint=endpoint,
@@ -246,4 +330,10 @@ GPT_CLIENT = GPTclient(
     api_version=api_version,
     model_name=model_name,
     check_connection=False,
+    timeout=timeout,
 )
+
+
+if __name__ == "__main__":
+    response = GPT_CLIENT.query("What is the capital of China?")
+    print(f"Response: {response}")

embodied_gen/utils/gpt_config.yaml

@@ -1,5 +1,5 @@
 # config.yaml
-agent_type: "qwen2.5-vl" # gpt-4o or qwen2.5-vl
+agent_type: "gpt-5.4" # gpt-4o, gpt-5.4 or qwen2.5-vl
 
 gpt-4o:
   endpoint: https://xxx.openai.azure.com
@@ -7,6 +7,12 @@ gpt-4o:
   api_version: 2025-xx-xx
   model_name: yfb-gpt-4o
 
+gpt-5.4:
+  endpoint: https://yfb-openai-sweden.openai.azure.com/
+  api_key: xxx
+  api_version: 2024-12-01-preview
+  model_name: gpt-5.4
+
 qwen2.5-vl:
   endpoint: https://openrouter.ai/api/v1
   api_key: sk-or-v1-xxx