        match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)

        if match:
            bid = match.group(1)
            action = f'click("{bid}")'
        else:
            raise Exception("Can't find the button's bid")

        answer = f"""I'm clicking the button as requested.
<action>
{action}
</action>
"""
        return dict(role="assistant", content=answer)

    def get_stats(self):
        return {}

@dataclass
class CheatMiniWoBLLMArgs_ParseRetry(BaseModelArgs):
    n_retry: int = 2
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatMiniWoBLLM_ParseRetry(n_retry=self.n_retry)

@dataclass
class CheatLLM_LLMError:
    """For unit-testing purposes only. Fails to call LLM"""

    n_retry: int = 0
    success: bool = False

    def __call__(self, messages) -> str:
        if self.success:
            if isinstance(messages, Discussion):
                prompt = messages.to_string()
            else:
                prompt = messages[1].get("content", "")
            match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)

            if match:
                bid = match.group(1)
                action = f'click("{bid}")'


<action>
{action}
</action>
"""
        return dict(role="assistant", content=answer)

    def get_stats(self):
        return {}

@dataclass
class CheatMiniWoBLLMArgs_ParseRetry(BaseModelArgs):
    n_retry: int = 2
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatMiniWoBLLM_ParseRetry(n_retry=self.n_retry)

@dataclass
class CheatLLM_LLMError:
    """For unit-testing purposes only. Fails to call LLM"""

    n_retry: int = 0
    success: bool = False

    def __call__(self, messages) -> str:
        if self.success:
            if isinstance(messages, Discussion):
                prompt = messages.to_string()
            else:
                prompt = messages[1].get("content", "")
            match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)

            if match:
                bid = match.group(1)
                action = f'click("{bid}")'
            else:
                raise Exception("Can't find the button's bid")

            answer = f"""I'm clicking the button as requested.
    <action>
    {action}
    </action>
    """
            return dict(role="assistant", content=answer)
        raise OpenAIError("LLM failed to respond")

    def get_stats(self):
        return {"n_llm_retry": self.n_retry, "n_llm_busted_retry": int(not self.success)}

@dataclass
class CheatLLMArgs_LLMError(BaseModelArgs):
    n_retry: int = 2
    success: bool = False
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatLLM_LLMError(
            n_retry=self.n_retry,
            success=self.success,
        )

def test_generic_agent_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=2),
            flags=FLAGS_GPT_3_5,


            if match:
                bid = match.group(1)
                action = f'click("{bid}")'
            else:
                raise Exception("Can't find the button's bid")

            answer = f"""I'm clicking the button as requested.
    <action>
    {action}
    </action>
    """
            return dict(role="assistant", content=answer)
        raise OpenAIError("LLM failed to respond")

    def get_stats(self):
        return {"n_llm_retry": self.n_retry, "n_llm_busted_retry": int(not self.success)}

@dataclass
class CheatLLMArgs_LLMError(BaseModelArgs):
    n_retry: int = 2
    success: bool = False
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatLLM_LLMError(
            n_retry=self.n_retry,
            success=self.success,
        )

def test_generic_agent_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=2),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        # TODO why these tests don't work with ray backend?
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)
        print(result_record)
        target = {
            "stats.cum_n_retry": 2,


            return dict(role="assistant", content=answer)
        raise OpenAIError("LLM failed to respond")

    def get_stats(self):
        return {"n_llm_retry": self.n_retry, "n_llm_busted_retry": int(not self.success)}

@dataclass
class CheatLLMArgs_LLMError(BaseModelArgs):
    n_retry: int = 2
    success: bool = False
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatLLM_LLMError(
            n_retry=self.n_retry,
            success=self.success,
        )

def test_generic_agent_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=2),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        # TODO why these tests don't work with ray backend?
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)
        print(result_record)
        target = {
            "stats.cum_n_retry": 2,
            "stats.cum_busted_retry": 0,
            "n_steps": 1,
            "cum_reward": 1.0,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_bust_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=10),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "stats.cum_n_retry": 5,
            "stats.cum_busted_retry": 1,


    with tempfile.TemporaryDirectory() as tmp_dir:
        # TODO why these tests don't work with ray backend?
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)
        print(result_record)
        target = {
            "stats.cum_n_retry": 2,
            "stats.cum_busted_retry": 0,
            "n_steps": 1,
            "cum_reward": 1.0,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_bust_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=10),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "stats.cum_n_retry": 5,
            "stats.cum_busted_retry": 1,
            "n_steps": 0,
            "cum_reward": 0,
            "err_msg": None,  # parsing error is considered an agent failure, not a code error
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_llm_error_success():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatLLMArgs_LLMError(n_retry=3, success=True),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "stats.cum_n_llm_retry": 3,
            "n_steps": 1,


    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "stats.cum_n_retry": 5,
            "stats.cum_busted_retry": 1,
            "n_steps": 0,
            "cum_reward": 0,
            "err_msg": None,  # parsing error is considered an agent failure, not a code error
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_llm_error_success():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatLLMArgs_LLMError(n_retry=3, success=True),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "stats.cum_n_llm_retry": 3,
            "n_steps": 1,
            "cum_reward": 1.0,
            "err_msg": None,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_llm_error_no_success():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatLLMArgs_LLMError(n_retry=5, success=False),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "n_steps": 0,
            "cum_reward": 0,


    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "stats.cum_n_llm_retry": 3,
            "n_steps": 1,
            "cum_reward": 1.0,
            "err_msg": None,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_llm_error_no_success():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatLLMArgs_LLMError(n_retry=5, success=False),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "n_steps": 0,
            "cum_reward": 0,
            "err_msg": "Exception uncaught by agent or environment in task miniwob.click-test.\nOpenAIError:\nLLM failed to respond",
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

if __name__ == "__main__":
    # test_generic_agent()
    test_generic_agent_parse_retry()


    def get_stats(self):
        return {}

@dataclass
class CheatMiniWoBLLMArgs_ParseRetry(BaseModelArgs):
    n_retry: int = 2
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatMiniWoBLLM_ParseRetry(n_retry=self.n_retry)

@dataclass
class CheatLLM_LLMError:
    """For unit-testing purposes only. Fails to call LLM"""

    n_retry: int = 0
    success: bool = False

    def __call__(self, messages) -> str:
        if self.success:
            if isinstance(messages, Discussion):
                prompt = messages.to_string()
            else:
                prompt = messages[1].get("content", "")
            match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)

            if match:
                bid = match.group(1)
                action = f'click("{bid}")'
            else:
                raise Exception("Can't find the button's bid")

            answer = f"""I'm clicking the button as requested.
    <action>
    {action}
    </action>
    """
            return dict(role="assistant", content=answer)
        raise OpenAIError("LLM failed to respond")

    def get_stats(self):
        return {"n_llm_retry": self.n_retry, "n_llm_busted_retry": int(not self.success)}

@dataclass
class CheatLLMArgs_LLMError(BaseModelArgs):
    n_retry: int = 2
    success: bool = False
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatLLM_LLMError(
            n_retry=self.n_retry,
            success=self.success,
        )

def test_generic_agent_parse_retry():
    exp_args = ExpArgs(


            if isinstance(messages, Discussion):
                prompt = messages.to_string()
            else:
                prompt = messages[1].get("content", "")
            match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)

            if match:
                bid = match.group(1)
                action = f'click("{bid}")'
            else:
                raise Exception("Can't find the button's bid")

            answer = f"""I'm clicking the button as requested.
    <action>
    {action}
    </action>
    """
            return dict(role="assistant", content=answer)
        raise OpenAIError("LLM failed to respond")

    def get_stats(self):
        return {"n_llm_retry": self.n_retry, "n_llm_busted_retry": int(not self.success)}

@dataclass
class CheatLLMArgs_LLMError(BaseModelArgs):
    n_retry: int = 2
    success: bool = False
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatLLM_LLMError(
            n_retry=self.n_retry,
            success=self.success,
        )

def test_generic_agent_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=2),
            flags=FLAGS_GPT_3_5,


                raise Exception("Can't find the button's bid")

            answer = f"""I'm clicking the button as requested.
    <action>
    {action}
    </action>
    """
            return dict(role="assistant", content=answer)
        raise OpenAIError("LLM failed to respond")

    def get_stats(self):
        return {"n_llm_retry": self.n_retry, "n_llm_busted_retry": int(not self.success)}

@dataclass
class CheatLLMArgs_LLMError(BaseModelArgs):
    n_retry: int = 2
    success: bool = False
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatLLM_LLMError(
            n_retry=self.n_retry,
            success=self.success,
        )

def test_generic_agent_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=2),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        # TODO why these tests don't work with ray backend?
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)
        print(result_record)
        target = {
            "stats.cum_n_retry": 2,


import logging
import tempfile

import pytest

from agentlab.agents.visualwebarena.agent import VisualWebArenaAgentArgs
from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT

@pytest.mark.pricy
def test_agent():
    with tempfile.TemporaryDirectory() as exp_dir:
        env_args = EnvArgs(
            task_name="miniwob.click-button",
            task_seed=0,
            max_steps=10,
            headless=True,
        )

        chat_model_args = CHAT_MODEL_ARGS_DICT["openai/gpt-4o-mini-2024-07-18"]

        exp_args = [
            ExpArgs(
                agent_args=VisualWebArenaAgentArgs(
                    temperature=0.1,
                    chat_model_args=chat_model_args,
                ),
                env_args=env_args,
                logging_level=logging.INFO,
            ),
            ExpArgs(
                agent_args=VisualWebArenaAgentArgs(
                    temperature=0.0,
                    chat_model_args=chat_model_args,
                ),
                env_args=env_args,
                logging_level=logging.INFO,
            ),
        ]

        for exp_arg in exp_args:
            exp_arg.agent_args.prepare()
            exp_arg.prepare(exp_dir)

        for exp_arg in exp_args:
            exp_arg.run()
            exp_arg.agent_args.close()


import os
import uuid
from pathlib import Path

try:
    from tapeagents.steps import ImageObservation

    from agentlab.agents.tapeagent.agent import TapeAgent, TapeAgentArgs, load_config
    from agentlab.benchmarks.gaia import GaiaBenchmark, GaiaQuestion
except ModuleNotFoundError:
    import pytest

    pytest.skip("Skipping test due to missing dependencies", allow_module_level=True)

def mock_dataset() -> dict:
    """Mock dataset for testing purposes."""
    data = [{"task_id": str(uuid.uuid4()), "file_name": "", "file_path": ""} for i in range(165)]
    data[5] = {
        "task_id": "32102e3e-d12a-4209-9163-7b3a104efe5d",
        "Question": """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet.""",
        "Level": "2",
        "Final answer": "Time-Parking 2: Parallel Universe",
        "file_name": "32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx",
        "file_path": "tests/data/32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx",
        "Annotator Metadata": {
            "Steps": """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe.""",
            "Number of steps": "3",
            "How long did this take?": "1 minute",
            "Tools": "1. Microsoft Excel",
            "Number of tools": "1",
        },
    }
    data[20] = {
        "task_id": "df6561b2-7ee5-4540-baab-5095f742716a",
        "Question": "When you take the average of the standard population deviation of the red numbers and the standard sample deviation of the green numbers in this image using the statistics module in Python 3.11, what is the result rounded to the nearest three decimal points?",
        "Level": "2",
        "Final answer": "17.056",
        "file_name": "df6561b2-7ee5-4540-baab-5095f742716a.png",
        "file_path": "tests/data/df6561b2-7ee5-4540-baab-5095f742716a.png",
        "Annotator Metadata": {
            "Steps": "1. Opened the PNG file.\n2. Made separate lists of the red numbers and green numbers.\n3. Opened a Python compiler.\n4. Ran the following code:\n```\nimport statistics as st\nred = st.pstdev([24, 74, 28, 54, 73, 33, 64, 73, 60, 53, 59, 40, 65, 76, 48, 34, 62, 70, 31, 24, 51, 55, 78, 76, 41, 77, 51])\ngreen = st.stdev([39, 29, 28, 72, 68, 47, 64, 74, 72, 40, 75, 26, 27, 37, 31, 55, 44, 64, 65, 38, 46, 66, 35, 76, 61, 53, 49])\navg = st.mean([red, green])\nprint(avg)\n```\n5. Rounded the output.",
            "Number of steps": "5",
            "How long did this take?": "20 minutes",
            "Tools": "1. Python compiler\n2. Image recognition tools",
            "Number of tools": "2",
        },
    }
    return {"validation": data}

def test_agent_creation():
    config = load_config("gaia_val")
    args = TapeAgentArgs(config=config)
    agent = args.make_agent()
    assert isinstance(agent, TapeAgent)
    assert agent.agent.name == "gaia_agent"

def test_gaia_bench():
    config = load_config("gaia_val")
    bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
    assert bench.name == "gaia"
    assert bench.split == "validation"
    assert len(bench.env_args_list) == 165

    task = bench.env_args_list[5].task
    question = """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet."""
    steps = """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe."""
    assert task["task_id"] == "32102e3e-d12a-4209-9163-7b3a104efe5d"
    assert task["Question"] == question
    assert task["Level"] == "2"
    assert task["Final answer"] == "Time-Parking 2: Parallel Universe"
    assert task["file_name"] == "32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx"
    assert task["Annotator Metadata"]["Steps"] == steps
    assert task["Annotator Metadata"]["Number of steps"] == "3"
    assert task["Annotator Metadata"]["How long did this take?"] == "1 minute"
    assert task["Annotator Metadata"]["Tools"] == "1. Microsoft Excel"
    assert task["Annotator Metadata"]["Number of tools"] == "1"

def test_gaia_gym_reset():
    exp_dir = "/tmp/gaia_unit_test"
    os.makedirs(exp_dir, exist_ok=True)

    config = load_config("gaia_val")
    bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
    args = bench.env_args_list[5]
    env = args.make_env(Path(exp_dir))
    steps, _ = env.reset()
    assert len(steps) == 1
    assert isinstance(steps[0], GaiaQuestion)
    assert steps[0].content.startswith(args.task["Question"])

    args = bench.env_args_list[20]
    env = args.make_env(Path(exp_dir))
    steps, _ = env.reset()
    assert len(steps) == 2
    assert isinstance(steps[0], GaiaQuestion)
    assert steps[0].content == args.task["Question"]
    assert isinstance(steps[1], ImageObservation)
    assert os.path.basename(steps[1].image_path) == args.task["file_name"]


import os
import uuid
from pathlib import Path

try:
    from tapeagents.steps import ImageObservation

    from agentlab.agents.tapeagent.agent import TapeAgent, TapeAgentArgs, load_config
    from agentlab.benchmarks.gaia import GaiaBenchmark, GaiaQuestion
except ModuleNotFoundError:
    import pytest

    pytest.skip("Skipping test due to missing dependencies", allow_module_level=True)

def mock_dataset() -> dict:
    """Mock dataset for testing purposes."""
    data = [{"task_id": str(uuid.uuid4()), "file_name": "", "file_path": ""} for i in range(165)]
    data[5] = {
        "task_id": "32102e3e-d12a-4209-9163-7b3a104efe5d",
        "Question": """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet.""",
        "Level": "2",
        "Final answer": "Time-Parking 2: Parallel Universe",
        "file_name": "32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx",
        "file_path": "tests/data/32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx",
        "Annotator Metadata": {
            "Steps": """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe.""",
            "Number of steps": "3",
            "How long did this take?": "1 minute",
            "Tools": "1. Microsoft Excel",
            "Number of tools": "1",
        },
    }
    data[20] = {
        "task_id": "df6561b2-7ee5-4540-baab-5095f742716a",
        "Question": "When you take the average of the standard population deviation of the red numbers and the standard sample deviation of the green numbers in this image using the statistics module in Python 3.11, what is the result rounded to the nearest three decimal points?",
        "Level": "2",
        "Final answer": "17.056",
        "file_name": "df6561b2-7ee5-4540-baab-5095f742716a.png",
        "file_path": "tests/data/df6561b2-7ee5-4540-baab-5095f742716a.png",
        "Annotator Metadata": {
            "Steps": "1. Opened the PNG file.\n2. Made separate lists of the red numbers and green numbers.\n3. Opened a Python compiler.\n4. Ran the following code:\n```\nimport statistics as st\nred = st.pstdev([24, 74, 28, 54, 73, 33, 64, 73, 60, 53, 59, 40, 65, 76, 48, 34, 62, 70, 31, 24, 51, 55, 78, 76, 41, 77, 51])\ngreen = st.stdev([39, 29, 28, 72, 68, 47, 64, 74, 72, 40, 75, 26, 27, 37, 31, 55, 44, 64, 65, 38, 46, 66, 35, 76, 61, 53, 49])\navg = st.mean([red, green])\nprint(avg)\n```\n5. Rounded the output.",
            "Number of steps": "5",
            "How long did this take?": "20 minutes",
            "Tools": "1. Python compiler\n2. Image recognition tools",
            "Number of tools": "2",
        },
    }
    return {"validation": data}

def test_agent_creation():
    config = load_config("gaia_val")
    args = TapeAgentArgs(config=config)
    agent = args.make_agent()
    assert isinstance(agent, TapeAgent)
    assert agent.agent.name == "gaia_agent"

def test_gaia_bench():
    config = load_config("gaia_val")
    bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
    assert bench.name == "gaia"
    assert bench.split == "validation"
    assert len(bench.env_args_list) == 165

    task = bench.env_args_list[5].task
    question = """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet."""
    steps = """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe."""


        },
    }
    data[20] = {
        "task_id": "df6561b2-7ee5-4540-baab-5095f742716a",
        "Question": "When you take the average of the standard population deviation of the red numbers and the standard sample deviation of the green numbers in this image using the statistics module in Python 3.11, what is the result rounded to the nearest three decimal points?",
        "Level": "2",
        "Final answer": "17.056",
        "file_name": "df6561b2-7ee5-4540-baab-5095f742716a.png",
        "file_path": "tests/data/df6561b2-7ee5-4540-baab-5095f742716a.png",
        "Annotator Metadata": {
            "Steps": "1. Opened the PNG file.\n2. Made separate lists of the red numbers and green numbers.\n3. Opened a Python compiler.\n4. Ran the following code:\n```\nimport statistics as st\nred = st.pstdev([24, 74, 28, 54, 73, 33, 64, 73, 60, 53, 59, 40, 65, 76, 48, 34, 62, 70, 31, 24, 51, 55, 78, 76, 41, 77, 51])\ngreen = st.stdev([39, 29, 28, 72, 68, 47, 64, 74, 72, 40, 75, 26, 27, 37, 31, 55, 44, 64, 65, 38, 46, 66, 35, 76, 61, 53, 49])\navg = st.mean([red, green])\nprint(avg)\n```\n5. Rounded the output.",
            "Number of steps": "5",
            "How long did this take?": "20 minutes",
            "Tools": "1. Python compiler\n2. Image recognition tools",
            "Number of tools": "2",
        },
    }
    return {"validation": data}

def test_agent_creation():
    config = load_config("gaia_val")
    args = TapeAgentArgs(config=config)
    agent = args.make_agent()
    assert isinstance(agent, TapeAgent)
    assert agent.agent.name == "gaia_agent"

def test_gaia_bench():
    config = load_config("gaia_val")
    bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
    assert bench.name == "gaia"
    assert bench.split == "validation"
    assert len(bench.env_args_list) == 165

    task = bench.env_args_list[5].task
    question = """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet."""
    steps = """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe."""
    assert task["task_id"] == "32102e3e-d12a-4209-9163-7b3a104efe5d"
    assert task["Question"] == question
    assert task["Level"] == "2"
    assert task["Final answer"] == "Time-Parking 2: Parallel Universe"
    assert task["file_name"] == "32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx"
    assert task["Annotator Metadata"]["Steps"] == steps
    assert task["Annotator Metadata"]["Number of steps"] == "3"
    assert task["Annotator Metadata"]["How long did this take?"] == "1 minute"


        "file_path": "tests/data/df6561b2-7ee5-4540-baab-5095f742716a.png",
        "Annotator Metadata": {
            "Steps": "1. Opened the PNG file.\n2. Made separate lists of the red numbers and green numbers.\n3. Opened a Python compiler.\n4. Ran the following code:\n```\nimport statistics as st\nred = st.pstdev([24, 74, 28, 54, 73, 33, 64, 73, 60, 53, 59, 40, 65, 76, 48, 34, 62, 70, 31, 24, 51, 55, 78, 76, 41, 77, 51])\ngreen = st.stdev([39, 29, 28, 72, 68, 47, 64, 74, 72, 40, 75, 26, 27, 37, 31, 55, 44, 64, 65, 38, 46, 66, 35, 76, 61, 53, 49])\navg = st.mean([red, green])\nprint(avg)\n```\n5. Rounded the output.",
            "Number of steps": "5",
            "How long did this take?": "20 minutes",
            "Tools": "1. Python compiler\n2. Image recognition tools",
            "Number of tools": "2",
        },
    }
    return {"validation": data}

def test_agent_creation():
    config = load_config("gaia_val")
    args = TapeAgentArgs(config=config)
    agent = args.make_agent()
    assert isinstance(agent, TapeAgent)
    assert agent.agent.name == "gaia_agent"

def test_gaia_bench():
    config = load_config("gaia_val")
    bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
    assert bench.name == "gaia"
    assert bench.split == "validation"
    assert len(bench.env_args_list) == 165

    task = bench.env_args_list[5].task
    question = """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet."""
    steps = """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe."""
    assert task["task_id"] == "32102e3e-d12a-4209-9163-7b3a104efe5d"
    assert task["Question"] == question
    assert task["Level"] == "2"
    assert task["Final answer"] == "Time-Parking 2: Parallel Universe"
    assert task["file_name"] == "32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx"
    assert task["Annotator Metadata"]["Steps"] == steps
    assert task["Annotator Metadata"]["Number of steps"] == "3"
    assert task["Annotator Metadata"]["How long did this take?"] == "1 minute"
    assert task["Annotator Metadata"]["Tools"] == "1. Microsoft Excel"
    assert task["Annotator Metadata"]["Number of tools"] == "1"

def test_gaia_gym_reset():
    exp_dir = "/tmp/gaia_unit_test"
    os.makedirs(exp_dir, exist_ok=True)

    config = load_config("gaia_val")
    bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
    args = bench.env_args_list[5]
    env = args.make_env(Path(exp_dir))
    steps, _ = env.reset()
    assert len(steps) == 1
    assert isinstance(steps[0], GaiaQuestion)
    assert steps[0].content.startswith(args.task["Question"])

    args = bench.env_args_list[20]
    env = args.make_env(Path(exp_dir))
    steps, _ = env.reset()
    assert len(steps) == 2
    assert isinstance(steps[0], GaiaQuestion)


    bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
    assert bench.name == "gaia"
    assert bench.split == "validation"
    assert len(bench.env_args_list) == 165

    task = bench.env_args_list[5].task
    question = """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet."""
    steps = """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe."""
    assert task["task_id"] == "32102e3e-d12a-4209-9163-7b3a104efe5d"
    assert task["Question"] == question
    assert task["Level"] == "2"
    assert task["Final answer"] == "Time-Parking 2: Parallel Universe"
    assert task["file_name"] == "32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx"
    assert task["Annotator Metadata"]["Steps"] == steps
    assert task["Annotator Metadata"]["Number of steps"] == "3"
    assert task["Annotator Metadata"]["How long did this take?"] == "1 minute"
    assert task["Annotator Metadata"]["Tools"] == "1. Microsoft Excel"
    assert task["Annotator Metadata"]["Number of tools"] == "1"

def test_gaia_gym_reset():
    exp_dir = "/tmp/gaia_unit_test"
    os.makedirs(exp_dir, exist_ok=True)

    config = load_config("gaia_val")
    bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
    args = bench.env_args_list[5]
    env = args.make_env(Path(exp_dir))
    steps, _ = env.reset()
    assert len(steps) == 1
    assert isinstance(steps[0], GaiaQuestion)
    assert steps[0].content.startswith(args.task["Question"])

    args = bench.env_args_list[20]
    env = args.make_env(Path(exp_dir))
    steps, _ = env.reset()
    assert len(steps) == 2
    assert isinstance(steps[0], GaiaQuestion)
    assert steps[0].content == args.task["Question"]
    assert isinstance(steps[1], ImageObservation)
    assert os.path.basename(steps[1].image_path) == args.task["file_name"]


from copy import deepcopy

import bgym
import pytest
from bgym import HighLevelActionSet, HighLevelActionSetArgs

from agentlab.agents import dynamic_prompting as dp
from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_3_5
from agentlab.agents.generic_agent.generic_agent_prompt import GenericPromptFlags, MainPrompt
from agentlab.llm.llm_utils import count_tokens

html_template = """

Hello World.
Step {}.

some extra text to make the html longer

"""

base_obs = {
    "goal": "do this and that",
    "goal_object": [{"type": "text", "text": "do this and that"}],
    "chat_messages": [{"role": "user", "message": "do this and that"}],
    "axtree_txt": "[1] Click me",
    "focused_element_bid": "45-256",
    "open_pages_urls": ["https://example.com"],
    "open_pages_titles": ["Example"],
    "active_page_index": 0,
}

OBS_HISTORY = [
    base_obs
    | {
        "pruned_html": html_template.format(1),
        "last_action_error": "",
    },
    base_obs
    | {
        "pruned_html": html_template.format(2),
        "last_action_error": "Hey, this is an error in the past",
    },
    base_obs
    | {
        "pruned_html": html_template.format(3),
        "last_action_error": "Hey, there is an error now",
    },
]
ACTIONS = ["click('41')", "click('42')"]
MEMORIES = ["memory A", "memory B"]
THOUGHTS = ["thought A", "thought B"]

ALL_TRUE_FLAGS = GenericPromptFlags(
    obs=dp.ObsFlags(
        use_html=True,
        use_ax_tree=True,
        use_tabs=True,
        use_focused_element=True,
        use_error_logs=True,
        use_history=True,
        use_past_error_logs=True,
        use_action_history=True,
        use_think_history=True,
        use_diff=True,
        html_type="pruned_html",
        use_screenshot=False,  # TODO test this
        use_som=False,  # TODO test this
        extract_visible_tag=True,
        extract_clickable_tag=True,
        extract_coords=False,
        filter_visible_elements_only=True,
    ),
    action=dp.ActionFlags(
        action_set=HighLevelActionSetArgs(
            subsets=["bid"],
            multiaction=True,
        ),
        long_description=True,
        individual_examples=True,
    ),
    use_plan=True,
    use_criticise=True,
    use_thinking=True,
    use_memory=True,
    use_concrete_example=True,
    use_abstract_example=True,
    use_hints=True,
    enable_chat=False,  # TODO test this
    max_prompt_tokens=None,
    be_cautious=True,
    extra_instructions=None,
)

FLAG_EXPECTED_PROMPT = [
    (
        "obs.use_html",
        ("HTML:", "", "Hello World.", "Step 3."),  # last obs will be in obs
    ),
    (
        "obs.use_ax_tree",
        ("AXTree:", "Click me"),
    ),
    (
        "obs.use_tabs",
        ("Currently open tabs:", "(active tab)"),
    ),
    (
        "obs.use_focused_element",
        ("Focused element:", "bid='45-256'"),
    ),
    (
        "obs.use_error_logs",
        ("Hey, there is an error now",),
    ),
    (
        "use_plan",
        ("You just executed step", "1- think\n2- do it"),
    ),
    (
        "use_criticise",
        (
            "Criticise action_draft",
            "<criticise>",
            "</criticise>",
            "<action_draft>",
        ),
    ),
    (
        "use_thinking",
        ("<think>", "</think>"),
    ),
    (
        "obs.use_past_error_logs",
        ("Hey, this is an error in the past",),
    ),
    (
        "obs.use_action_history",
        ("<action>", "click('41')", "click('42')"),
    ),
    (
        "use_memory",
        ("<memory>", "</memory>", "memory A", "memory B"),
    ),
    # (
    #     "obs.use_diff",
    #     ("diff:", "- Step 2", "Identical"),
    # ),
    (
        "use_concrete_example",
        ("# Concrete Example", "<action>\nclick('a324')"),
    ),
    (
        "use_abstract_example",
        ("# Abstract Example",),
    ),
    # (
    #     "action.action_set.multiaction",
    #     ("One or several actions, separated by new lines",),
    # ),
]

def test_shrinking_observation():
    flags = deepcopy(FLAGS_GPT_3_5)
    flags.obs.use_html = True

    prompt_maker = MainPrompt(
        action_set=HighLevelActionSet(),
        obs_history=OBS_HISTORY,
        actions=ACTIONS,
        memories=MEMORIES,
        thoughts=THOUGHTS,
        previous_plan="1- think\n2- do it",
        step=2,
        flags=flags,
    )

    prompt = str(prompt_maker.prompt)
    new_prompt = str(
        dp.fit_tokens(prompt_maker, max_prompt_tokens=count_tokens(prompt) - 1, max_iterations=7)
    )
    assert count_tokens(new_prompt) < count_tokens(prompt)
    assert "[1] Click me" in prompt
    assert "[1] Click me" in new_prompt
    assert "" in prompt
    assert "" not in new_prompt

@pytest.mark.parametrize("flag_name, expected_prompts", FLAG_EXPECTED_PROMPT)
def test_main_prompt_elements_gone_one_at_a_time(flag_name: str, expected_prompts):

    if flag_name in ["use_thinking", "obs.use_action_history"]:
        return  # TODO design new tests for those two flags

    # Disable the flag
    flags = deepcopy(ALL_TRUE_FLAGS)
    if "." in flag_name:
        prefix, flag_name = flag_name.split(".")
        sub_flags = getattr(flags, prefix)
        setattr(sub_flags, flag_name, False)
    else:
        setattr(flags, flag_name, False)

    if flag_name == "use_memory":
        memories = None
    else:
        memories = MEMORIES

    # Initialize MainPrompt
    prompt = str(
        MainPrompt(
            action_set=flags.action.action_set.make_action_set(),
            obs_history=OBS_HISTORY,
            actions=ACTIONS,
            memories=memories,
            thoughts=THOUGHTS,
            previous_plan="1- think\n2- do it",
            step=2,
            flags=flags,
        ).prompt
    )

    # Verify all elements are not present
    for expected in expected_prompts:
        assert expected not in prompt

def test_main_prompt_elements_present():
    # Make sure the flag is enabled

    # Initialize MainPrompt
    prompt = str(
        MainPrompt(
            action_set=HighLevelActionSet(),
            obs_history=OBS_HISTORY,
            actions=ACTIONS,
            memories=MEMORIES,
            thoughts=THOUGHTS,
            previous_plan="1- think\n2- do it",
            step=2,
            flags=ALL_TRUE_FLAGS,
        ).prompt
    )
    # Verify all elements are not present
    for _, expected_prompts in FLAG_EXPECTED_PROMPT:
        for expected in expected_prompts:
            assert expected in prompt

if __name__ == "__main__":
    # for debugging
    test_shrinking_observation()
    test_main_prompt_elements_present()
    # for flag, expected_prompts in FLAG_EXPECTED_PROMPT:
    #     test_main_prompt_elements_gone_one_at_a_time(flag, expected_prompts)


),
    # (
    #     "obs.use_diff",
    #     ("diff:", "- Step 2", "Identical"),
    # ),
    (
        "use_concrete_example",
        ("# Concrete Example", "<action>\nclick('a324')"),
    ),
    (
        "use_abstract_example",
        ("# Abstract Example",),
    ),
    # (
    #     "action.action_set.multiaction",
    #     ("One or several actions, separated by new lines",),
    # ),
]

def test_shrinking_observation():
    flags = deepcopy(FLAGS_GPT_3_5)
    flags.obs.use_html = True

    prompt_maker = MainPrompt(
        action_set=HighLevelActionSet(),
        obs_history=OBS_HISTORY,
        actions=ACTIONS,
        memories=MEMORIES,
        thoughts=THOUGHTS,
        previous_plan="1- think\n2- do it",
        step=2,
        flags=flags,
    )

    prompt = str(prompt_maker.prompt)
    new_prompt = str(
        dp.fit_tokens(prompt_maker, max_prompt_tokens=count_tokens(prompt) - 1, max_iterations=7)
    )
    assert count_tokens(new_prompt) < count_tokens(prompt)
    assert "[1] Click me" in prompt
    assert "[1] Click me" in new_prompt
    assert "" in prompt
    assert "" not in new_prompt

@pytest.mark.parametrize("flag_name, expected_prompts", FLAG_EXPECTED_PROMPT)
def test_main_prompt_elements_gone_one_at_a_time(flag_name: str, expected_prompts):

    if flag_name in ["use_thinking", "obs.use_action_history"]:
        return  # TODO design new tests for those two flags

    # Disable the flag
    flags = deepcopy(ALL_TRUE_FLAGS)
    if "." in flag_name:
        prefix, flag_name = flag_name.split(".")
        sub_flags = getattr(flags, prefix)
        setattr(sub_flags, flag_name, False)
    else:
        setattr(flags, flag_name, False)

    if flag_name == "use_memory":
        memories = None
    else:


actions=ACTIONS,
        memories=MEMORIES,
        thoughts=THOUGHTS,
        previous_plan="1- think\n2- do it",
        step=2,
        flags=flags,
    )

    prompt = str(prompt_maker.prompt)
    new_prompt = str(
        dp.fit_tokens(prompt_maker, max_prompt_tokens=count_tokens(prompt) - 1, max_iterations=7)
    )
    assert count_tokens(new_prompt) < count_tokens(prompt)
    assert "[1] Click me" in prompt
    assert "[1] Click me" in new_prompt
    assert "" in prompt
    assert "" not in new_prompt

@pytest.mark.parametrize("flag_name, expected_prompts", FLAG_EXPECTED_PROMPT)
def test_main_prompt_elements_gone_one_at_a_time(flag_name: str, expected_prompts):

    if flag_name in ["use_thinking", "obs.use_action_history"]:
        return  # TODO design new tests for those two flags

    # Disable the flag
    flags = deepcopy(ALL_TRUE_FLAGS)
    if "." in flag_name:
        prefix, flag_name = flag_name.split(".")
        sub_flags = getattr(flags, prefix)
        setattr(sub_flags, flag_name, False)
    else:
        setattr(flags, flag_name, False)

    if flag_name == "use_memory":
        memories = None
    else:
        memories = MEMORIES

    # Initialize MainPrompt
    prompt = str(
        MainPrompt(
            action_set=flags.action.action_set.make_action_set(),
            obs_history=OBS_HISTORY,
            actions=ACTIONS,
            memories=memories,
            thoughts=THOUGHTS,
            previous_plan="1- think\n2- do it",
            step=2,
            flags=flags,
        ).prompt
    )

    # Verify all elements are not present
    for expected in expected_prompts:
        assert expected not in prompt

def test_main_prompt_elements_present():
    # Make sure the flag is enabled

    # Initialize MainPrompt
    prompt = str(
        MainPrompt(
            action_set=HighLevelActionSet(),
            obs_history=OBS_HISTORY,
            actions=ACTIONS,
            memories=MEMORIES,
            thoughts=THOUGHTS,
            previous_plan="1- think\n2- do it",
            step=2,
            flags=ALL_TRUE_FLAGS,
        ).prompt
    )
    # Verify all elements are not present
    for _, expected_prompts in FLAG_EXPECTED_PROMPT:


    # Initialize MainPrompt
    prompt = str(
        MainPrompt(
            action_set=flags.action.action_set.make_action_set(),
            obs_history=OBS_HISTORY,
            actions=ACTIONS,
            memories=memories,
            thoughts=THOUGHTS,
            previous_plan="1- think\n2- do it",
            step=2,
            flags=flags,
        ).prompt
    )

    # Verify all elements are not present
    for expected in expected_prompts:
        assert expected not in prompt

def test_main_prompt_elements_present():
    # Make sure the flag is enabled

    # Initialize MainPrompt
    prompt = str(
        MainPrompt(
            action_set=HighLevelActionSet(),
            obs_history=OBS_HISTORY,
            actions=ACTIONS,
            memories=MEMORIES,
            thoughts=THOUGHTS,
            previous_plan="1- think\n2- do it",
            step=2,
            flags=ALL_TRUE_FLAGS,
        ).prompt
    )
    # Verify all elements are not present
    for _, expected_prompts in FLAG_EXPECTED_PROMPT:
        for expected in expected_prompts:
            assert expected in prompt

if __name__ == "__main__":
    # for debugging
    test_shrinking_observation()
    test_main_prompt_elements_present()
    # for flag, expected_prompts in FLAG_EXPECTED_PROMPT:
    #     test_main_prompt_elements_gone_one_at_a_time(flag, expected_prompts)


docs.source.conf

# Configuration file for the Sphinx documentation builder.
#
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information

import os
import subprocess

# Automatically retrieve the project version from Git
def get_version():
    try:
        return subprocess.check_output(["git", "describe", "--tags"], encoding="utf-8").strip()
    except Exception:
        return "0.0.0"

project = "AgentLab"
author = "ServiceNow"
release = get_version()  # Full version string including tags
version = release  # Short version (e.g., 1.0)

# -- General configuration ---------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
extensions = [
    "sphinx.ext.duration",
    "sphinx.ext.doctest",
    "sphinx.ext.autodoc",
    "sphinx.ext.autosummary",
    "sphinx.ext.intersphinx",
    # "myst_parser",  # Add this to enable Markdown parsing
    "sphinx.ext.napoleon",
]

intersphinx_mapping = {
    "rtd": ("https://docs.readthedocs.io/en/stable/", None),
    "python": ("https://docs.python.org/3/", None),
    "sphinx": ("https://www.sphinx-doc.org/en/master/", None),
}
intersphinx_disabled_domains = ["std"]

autodoc_default_options = {
    "members": True,
    "undoc-members": True,
    "show-inheritance": True,
}

source_suffix = {
    ".rst": "restructuredtext",
}

templates_path = ["_templates"]
exclude_patterns = []

# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output

html_theme = "sphinx_rtd_theme"
html_theme_options = {
    "navigation_depth": -1,
    "collapse_navigation": False,
    "display_version": True,
    "version_selector": True,
}
html_static_path = ["_static"]

import sys

sys.path.insert(0, os.path.abspath("../../src"))


# Configuration file for the Sphinx documentation builder.
#
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information

import os
import subprocess

# Automatically retrieve the project version from Git
def get_version():
    try:
        return subprocess.check_output(["git", "describe", "--tags"], encoding="utf-8").strip()
    except Exception:
        return "0.0.0"

project = "AgentLab"
author = "ServiceNow"
release = get_version()  # Full version string including tags
version = release  # Short version (e.g., 1.0)

# -- General configuration ---------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
extensions = [
    "sphinx.ext.duration",
    "sphinx.ext.doctest",
    "sphinx.ext.autodoc",
    "sphinx.ext.autosummary",
    "sphinx.ext.intersphinx",
    # "myst_parser",  # Add this to enable Markdown parsing
    "sphinx.ext.napoleon",
]


from pathlib import Path

from bgym import DEFAULT_BENCHMARKS
from dotenv import load_dotenv

from agentlab.agents.generic_agent.tmlr_config import (
    BASE_FLAGS,
    CHAT_MODEL_ARGS_DICT,
    GenericAgentArgs,
)
from agentlab.benchmarks.setup_benchmark import ensure_benchmark
from agentlab.experiments.study import Study

# This ensures MiniWob assets are downloaded and sets the MINIWOB_URL .env in the project dir.
project_dir = Path(__file__).parents[2]
ensure_benchmark("miniwob", project_root=project_dir)
load_dotenv(project_dir.joinpath(".env"), override=False)  # load .env variables

agent_config = GenericAgentArgs(
    chat_model_args=CHAT_MODEL_ARGS_DICT[
        "openai/gpt-5-nano-2025-08-07"
    ],  # or CHAT_MODEL_ARGS_DICT["openrouter/openai/gpt-5-nano"]
    flags=BASE_FLAGS,
)

# choose a list of agents to evaluate
agent_configs = [agent_config]
# chose your benchmark
benchmark = DEFAULT_BENCHMARKS["miniwob_tiny_test"]()

# benchmark = DEFAULT_BENCHMARKS["miniwob"]()  # 125 tasks
# benchmark = benchmark.subset_from_glob(column="task_name", glob="*enter*")

## Number of parallel jobs
n_jobs = 4  # Make sure to use 1 job when debugging in VSCode

if __name__ == "__main__":  # necessary for dask backend

    # A study evaluates multiple agents on a benchmark
    study = Study(agent_configs, benchmark)

    study.run(
        n_jobs=n_jobs,
        parallel_backend="ray",  # "ray", "joblib" or "sequential"
        n_relaunch=3,  # will automatically relaunch tasks with system error or incomplete tasks.
    )


import json
import logging
import os

from agentlab.agents.tool_use_agent.tool_use_agent import OSWORLD_CLAUDE
from agentlab.benchmarks.osworld import OsworldBenchmark
from agentlab.experiments.study import Study, make_study

fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

def get_most_recent_incomplete_study() -> Study:
    """
    Relaunch an existing study, this will continue incomplete experiments and relaunch errored experiments.
    """
    study = Study.load_most_recent()
    study.find_incomplete(include_errors=True)
    return study

def get_task_ids() -> set[str]:
    with open("experiments/osworld_debug_task_ids.json", "r") as f:
        task_ids = json.load(f)
    return set([task["id"] for task in task_ids])

def main():
    n_jobs = 4
    use_vmware = True
    relaunch = False
    agent_args = [
        OSWORLD_CLAUDE,
        #    OSWORLD_OAI # performs poorly.
    ]  # type: ignore
    parallel_backend = "ray"
    os.environ["AGENTLAB_DEBUG"] = os.environ.get("AGENTLAB_DEBUG", "1")

    study = make_study(
        benchmark=OsworldBenchmark(
            test_set_name="test_small.json"
        ),  # or test_all.json (Exper)  # type: ignore
        agent_args=agent_args,  # type: ignore
        comment="osworld debug 2",
        logging_level=logging.INFO,
        logging_level_stdout=logging.INFO,
    )

    if use_vmware:
        for exp_args in study.exp_args_list:
            exp_args.env_args.provider_name = "vmware"  # type: ignore
            exp_args.env_args.path_to_vm = "OSWorld/vmware_vm_data/Ubuntu0/Ubuntu0.vmx"  # type: ignore
        parallel_backend = "sequential"

    if os.environ.get("AGENTLAB_DEBUG"):
        task_ids = get_task_ids()
        study.exp_args_list = [exp_args for exp_args in study.exp_args_list if exp_args.env_args.task["id"] in task_ids]  # type: ignore
        print(f"Debug on {len(study.exp_args_list)} experiments")
        n_jobs = 1  # Make sure to use 1 job when debugging in VS

    study = get_most_recent_incomplete_study() if relaunch else study
    study.run(n_jobs=n_jobs, n_relaunch=1, parallel_backend=parallel_backend)

if __name__ == "__main__":
    main()


import json
import logging
import os

from agentlab.agents.tool_use_agent.tool_use_agent import OSWORLD_CLAUDE
from agentlab.benchmarks.osworld import OsworldBenchmark
from agentlab.experiments.study import Study, make_study

fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

def get_most_recent_incomplete_study() -> Study:
    """
    Relaunch an existing study, this will continue incomplete experiments and relaunch errored experiments.
    """
    study = Study.load_most_recent()
    study.find_incomplete(include_errors=True)
    return study

def get_task_ids() -> set[str]:
    with open("experiments/osworld_debug_task_ids.json", "r") as f:
        task_ids = json.load(f)
    return set([task["id"] for task in task_ids])

def main():
    n_jobs = 4
    use_vmware = True
    relaunch = False
    agent_args = [
        OSWORLD_CLAUDE,
        #    OSWORLD_OAI # performs poorly.
    ]  # type: ignore
    parallel_backend = "ray"
    os.environ["AGENTLAB_DEBUG"] = os.environ.get("AGENTLAB_DEBUG", "1")

    study = make_study(


import logging
import os

from agentlab.agents.tool_use_agent.tool_use_agent import OSWORLD_CLAUDE
from agentlab.benchmarks.osworld import OsworldBenchmark
from agentlab.experiments.study import Study, make_study

fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

def get_most_recent_incomplete_study() -> Study:
    """
    Relaunch an existing study, this will continue incomplete experiments and relaunch errored experiments.
    """
    study = Study.load_most_recent()
    study.find_incomplete(include_errors=True)
    return study

def get_task_ids() -> set[str]:
    with open("experiments/osworld_debug_task_ids.json", "r") as f:
        task_ids = json.load(f)
    return set([task["id"] for task in task_ids])

def main():
    n_jobs = 4
    use_vmware = True
    relaunch = False
    agent_args = [
        OSWORLD_CLAUDE,
        #    OSWORLD_OAI # performs poorly.
    ]  # type: ignore
    parallel_backend = "ray"
    os.environ["AGENTLAB_DEBUG"] = os.environ.get("AGENTLAB_DEBUG", "1")

    study = make_study(
        benchmark=OsworldBenchmark(
            test_set_name="test_small.json"
        ),  # or test_all.json (Exper)  # type: ignore
        agent_args=agent_args,  # type: ignore
        comment="osworld debug 2",
        logging_level=logging.INFO,


main

fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

def get_most_recent_incomplete_study() -> Study:
    """
    Relaunch an existing study, this will continue incomplete experiments and relaunch errored experiments.
    """
    study = Study.load_most_recent()
    study.find_incomplete(include_errors=True)
    return study

def get_task_ids() -> set[str]:
    with open("experiments/osworld_debug_task_ids.json", "r") as f:
        task_ids = json.load(f)
    return set([task["id"] for task in task_ids])

def main():
    n_jobs = 4
    use_vmware = True
    relaunch = False
    agent_args = [
        OSWORLD_CLAUDE,
        #    OSWORLD_OAI # performs poorly.
    ]  # type: ignore
    parallel_backend = "ray"
    os.environ["AGENTLAB_DEBUG"] = os.environ.get("AGENTLAB_DEBUG", "1")

    study = make_study(
        benchmark=OsworldBenchmark(
            test_set_name="test_small.json"
        ),  # or test_all.json (Exper)  # type: ignore
        agent_args=agent_args,  # type: ignore
        comment="osworld debug 2",
        logging_level=logging.INFO,
        logging_level_stdout=logging.INFO,
    )

    if use_vmware:
        for exp_args in study.exp_args_list:
            exp_args.env_args.provider_name = "vmware"  # type: ignore
            exp_args.env_args.path_to_vm = "OSWorld/vmware_vm_data/Ubuntu0/Ubuntu0.vmx"  # type: ignore
        parallel_backend = "sequential"

    if os.environ.get("AGENTLAB_DEBUG"):
        task_ids = get_task_ids()
        study.exp_args_list = [exp_args for exp_args in study.exp_args_list if exp_args.env_args.task["id"] in task_ids]  # type: ignore
        print(f"Debug on {len(study.exp_args_list)} experiments")
        n_jobs = 1  # Make sure to use 1 job when debugging in VS

    study = get_most_recent_incomplete_study() if relaunch else study
    study.run(n_jobs=n_jobs, n_relaunch=1, parallel_backend=parallel_backend)

if __name__ == "__main__":
    main()


import argparse

from agentlab.agents.agent_args import AgentArgs
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.launch_exp import import_object
from agentlab.experiments.loop import EnvArgs, ExpArgs

def make_exp_args(agent_args: AgentArgs, start_url: str) -> ExpArgs:
    try:
        agent_args.flags.action.demo_mode = "default"
    except AttributeError:
        pass

    if isinstance(agent_args, GenericAgentArgs):
        agent_args.flags.enable_chat = True

    exp_args = ExpArgs(
        agent_args=agent_args,
        env_args=EnvArgs(
            max_steps=1000,
            task_seed=None,
            task_name="openended",
            task_kwargs={
                "start_url": start_url,
            },
            headless=False,
            record_video=True,
            wait_for_user_message=True,
            viewport={"width": 1500, "height": 1280},
            slow_mo=1000,
        ),
    )

    return exp_args

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--agent_config",
        type=str,
        default="agentlab.agents.generic_agent.AGENT_GPT5_MINI",
        help="""Python path to the agent config. Defaults to : "agentlab.agents.generic_agent.AGENT_GPT5_MINI".""",
    )
    parser.add_argument(
        "--start_url",
        type=str,
        default="https://www.google.com",
        help="The start page of the agent. Defaults to https://www.google.com",
    )

    args, unknown = parser.parse_known_args()
    agent_args = import_object(args.agent_config)
    exp_args = make_exp_args(agent_args, args.start_url)
    exp_args.prepare(RESULTS_DIR / "ui_assistant_logs")
    exp_args.run()

if __name__ == "__main__":
    main()


import argparse

from agentlab.agents.agent_args import AgentArgs
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.launch_exp import import_object
from agentlab.experiments.loop import EnvArgs, ExpArgs

def make_exp_args(agent_args: AgentArgs, start_url: str) -> ExpArgs:
    try:
        agent_args.flags.action.demo_mode = "default"
    except AttributeError:
        pass

    if isinstance(agent_args, GenericAgentArgs):
        agent_args.flags.enable_chat = True

    exp_args = ExpArgs(
        agent_args=agent_args,
        env_args=EnvArgs(
            max_steps=1000,
            task_seed=None,
            task_name="openended",
            task_kwargs={
                "start_url": start_url,
            },
            headless=False,
            record_video=True,
            wait_for_user_message=True,
            viewport={"width": 1500, "height": 1280},
            slow_mo=1000,
        ),
    )

    return exp_args

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--agent_config",
        type=str,
        default="agentlab.agents.generic_agent.AGENT_GPT5_MINI",
        help="""Python path to the agent config. Defaults to : "agentlab.agents.generic_agent.AGENT_GPT5_MINI".""",
    )
    parser.add_argument(
        "--start_url",
        type=str,
        default="https://www.google.com",
        help="The start page of the agent. Defaults to https://www.google.com",
    )

    args, unknown = parser.parse_known_args()
    agent_args = import_object(args.agent_config)


    exp_args = ExpArgs(
        agent_args=agent_args,
        env_args=EnvArgs(
            max_steps=1000,
            task_seed=None,
            task_name="openended",
            task_kwargs={
                "start_url": start_url,
            },
            headless=False,
            record_video=True,
            wait_for_user_message=True,
            viewport={"width": 1500, "height": 1280},
            slow_mo=1000,
        ),
    )

    return exp_args

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--agent_config",
        type=str,
        default="agentlab.agents.generic_agent.AGENT_GPT5_MINI",
        help="""Python path to the agent config. Defaults to : "agentlab.agents.generic_agent.AGENT_GPT5_MINI".""",
    )
    parser.add_argument(
        "--start_url",
        type=str,
        default="https://www.google.com",
        help="The start page of the agent. Defaults to https://www.google.com",
    )

    args, unknown = parser.parse_known_args()
    agent_args = import_object(args.agent_config)
    exp_args = make_exp_args(agent_args, args.start_url)
    exp_args.prepare(RESULTS_DIR / "ui_assistant_logs")
    exp_args.run()

if __name__ == "__main__":
    main()


import os
from dataclasses import dataclass
from pathlib import Path

import pandas as pd
from tqdm import tqdm

from agentlab.analyze import inspect_results
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.study import Study

@dataclass
class StudyInfo:
    study_dir: Path
    study: Study
    summary_df: pd.DataFrame
    should_delete: bool = False
    reason: str = ""

def search_for_reasons_to_archive(result_dir: Path, min_study_size: int = 0) -> list[StudyInfo]:

    study_info_list = []
    study_dirs = list(result_dir.iterdir())
    progress = tqdm(study_dirs, desc="Processing studies")
    for study_dir in progress:

        progress.set_postfix({"study_dir": study_dir})
        if not study_dir.is_dir():
            progress.set_postfix({"status": "skipped"})
            continue

        try:
            study = Study.load(study_dir)
        except Exception:
            study = None
        # get summary*.csv files and find the most recent
        summary_files = list(study_dir.glob("summary*.csv"))

        if len(summary_files) != 0:
            most_recent_summary = max(summary_files, key=os.path.getctime)
            summary_df = pd.read_csv(most_recent_summary)

        else:
            try:
                result_df = inspect_results.load_result_df(study_dir, progress_fn=None)
                summary_df = inspect_results.summarize_study(result_df)
            except Exception as e:
                print(f"  Error processing {study_dir}: {e}")
                continue

        study_info = StudyInfo(
            study_dir=study_dir,
            study=study,
            summary_df=summary_df,
        )

        if len(study_info.summary_df) == 0:
            study_info.should_delete = True
            study_info.reason = "Empty summary DataFrame"

        n_completed, n_total, n_err = 0, 0, 0

        for _, row in study_info.summary_df.iterrows():
            n_comp, n_tot = row["n_completed"].split("/")
            n_completed += int(n_comp)
            n_total += int(n_tot)
            n_err += int(row.get("n_err"))

        n_finished = n_completed - n_err

        # print(summary_df)
        # print(f"  {n_completed} / {n_total}, {n_err} errors")

        if "miniwob-tiny-test" in study_dir.name:
            study_info.should_delete = True
            study_info.reason += "Miniwob tiny test\n"
        if n_total == 0:
            study_info.should_delete = True
            study_info.reason += "No tasks\n"
        if n_completed == 0:
            study_info.should_delete = True
            study_info.reason += "No tasks completed\n"
        if float(n_finished) / float(n_total) < 0.5:
            study_info.should_delete = True
            study_info.reason += f"Less than 50% tasks finished, n_err: {n_err}, n_total: {n_total}, n_finished: {n_finished}, n_completed: {n_completed}\n"

        if n_total <= min_study_size:
            study_info.should_delete = True
            study_info.reason += (
                f"Too few tasks. n_total ({n_total}) <= min_study_size ({min_study_size})\n"
            )

        study_info_list.append(study_info)
    return study_info_list

if __name__ == "__main__":
    study_list_info = search_for_reasons_to_archive(RESULTS_DIR, min_study_size=5)
    archive_dir = RESULTS_DIR.parent / "archived_agentlab_results"  # type: Path
    archive_dir.mkdir(parents=True, exist_ok=True)

    # Uncomment the line below to prevent moving studies to archive
    archive_dir = None

    for study_info in study_list_info:
        if not study_info.should_delete:
            continue

        print(f"Study: {study_info.study_dir.name}")
        print(f"  Reason: {study_info.reason}")
        print(study_info.summary_df)
        print()

        if archive_dir is not None:
            # move to new dir
            new_path = archive_dir / study_info.study_dir.name
            study_info.study_dir.rename(new_path)
            # save reason in a file
            reason_file = new_path / "reason_to_archive.txt"
            reason_file.write_text(study_info.reason)


StudyInfo

import os
from dataclasses import dataclass
from pathlib import Path

import pandas as pd
from tqdm import tqdm

from agentlab.analyze import inspect_results
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.study import Study

@dataclass
class StudyInfo:
    study_dir: Path
    study: Study
    summary_df: pd.DataFrame
    should_delete: bool = False
    reason: str = ""

def search_for_reasons_to_archive(result_dir: Path, min_study_size: int = 0) -> list[StudyInfo]:

    study_info_list = []
    study_dirs = list(result_dir.iterdir())
    progress = tqdm(study_dirs, desc="Processing studies")
    for study_dir in progress:

        progress.set_postfix({"study_dir": study_dir})
        if not study_dir.is_dir():
            progress.set_postfix({"status": "skipped"})
            continue

        try:
            study = Study.load(study_dir)
        except Exception:
            study = None
        # get summary*.csv files and find the most recent
        summary_files = list(study_dir.glob("summary*.csv"))


from dataclasses import dataclass
from pathlib import Path

import pandas as pd
from tqdm import tqdm

from agentlab.analyze import inspect_results
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.study import Study

@dataclass
class StudyInfo:
    study_dir: Path
    study: Study
    summary_df: pd.DataFrame
    should_delete: bool = False
    reason: str = ""

def search_for_reasons_to_archive(result_dir: Path, min_study_size: int = 0) -> list[StudyInfo]:

    study_info_list = []
    study_dirs = list(result_dir.iterdir())
    progress = tqdm(study_dirs, desc="Processing studies")
    for study_dir in progress:

        progress.set_postfix({"study_dir": study_dir})
        if not study_dir.is_dir():
            progress.set_postfix({"status": "skipped"})
            continue

        try:
            study = Study.load(study_dir)
        except Exception:
            study = None
        # get summary*.csv files and find the most recent
        summary_files = list(study_dir.glob("summary*.csv"))

        if len(summary_files) != 0:
            most_recent_summary = max(summary_files, key=os.path.getctime)
            summary_df = pd.read_csv(most_recent_summary)

        else:
            try:
                result_df = inspect_results.load_result_df(study_dir, progress_fn=None)
                summary_df = inspect_results.summarize_study(result_df)
            except Exception as e:
                print(f"  Error processing {study_dir}: {e}")
                continue

        study_info = StudyInfo(
            study_dir=study_dir,
            study=study,
            summary_df=summary_df,
        )

        if len(study_info.summary_df) == 0:
            study_info.should_delete = True
            study_info.reason = "Empty summary DataFrame"

        n_completed, n_total, n_err = 0, 0, 0

        for _, row in study_info.summary_df.iterrows():
            n_comp, n_tot = row["n_completed"].split("/")
            n_completed += int(n_comp)
            n_total += int(n_tot)
            n_err += int(row.get("n_err"))

        n_finished = n_completed - n_err

        # print(summary_df)
        # print(f"  {n_completed} / {n_total}, {n_err} errors")

        if "miniwob-tiny-test" in study_dir.name:
            study_info.should_delete = True
            study_info.reason += "Miniwob tiny test\n"
        if n_total == 0:
            study_info.should_delete = True
            study_info.reason += "No tasks\n"
        if n_completed == 0:
            study_info.should_delete = True
            study_info.reason += "No tasks completed\n"
        if float(n_finished) / float(n_total) < 0.5:
            study_info.should_delete = True
            study_info.reason += f"Less than 50% tasks finished, n_err: {n_err}, n_total: {n_total}, n_finished: {n_finished}, n_completed: {n_completed}\n"

        if n_total <= min_study_size:
            study_info.should_delete = True
            study_info.reason += (
                f"Too few tasks. n_total ({n_total}) <= min_study_size ({min_study_size})\n"
            )

        study_info_list.append(study_info)
    return study_info_list

if __name__ == "__main__":
    study_list_info = search_for_reasons_to_archive(RESULTS_DIR, min_study_size=5)
    archive_dir = RESULTS_DIR.parent / "archived_agentlab_results"  # type: Path
    archive_dir.mkdir(parents=True, exist_ok=True)

    # Uncomment the line below to prevent moving studies to archive
    archive_dir = None

    for study_info in study_list_info:
        if not study_info.should_delete:
            continue

        print(f"Study: {study_info.study_dir.name}")
        print(f"  Reason: {study_info.reason}")
        print(study_info.summary_df)
        print()

        if archive_dir is not None:


import ast
import inspect
import math
from dataclasses import dataclass
from typing import Any, Union

import matplotlib.pyplot as plt
import PIL
from browsergym.core.action.highlevel import ACTION_SUBSETS
from PIL import Image, ImageDraw

BGYM_FUNCTION_MAP = {}
for subset in ("bid", "coord"):
    for func in ACTION_SUBSETS[subset]:
        if func not in BGYM_FUNCTION_MAP:
            BGYM_FUNCTION_MAP[func.__name__] = func

@dataclass
class ArgInfo:
    function_name: str
    name: str
    value: Any
    type: str
    start_index: int
    stop_index: int

def parse_function_calls(code_string: str) -> list[ArgInfo]:
    """
    Parse a string containing multiple function calls and return a list of ArgInfo objects
    for all arguments in all function calls.

    Args:
        code_string: String containing function calls

    Returns:
        List of ArgInfo objects containing detailed information about each argument

    Example:
        >>> code = '''
        ... mouse_click(34, 59)
        ... fill("a234", "test")
        ... '''
        >>> result = parse_function_calls(code)
        >>> # Returns list of ArgInfo objects for each argument
    """
    result = []

    try:
        # Parse the code string into an AST
        tree = ast.parse(code_string)

        # Extract all function calls
        for node in ast.walk(tree):
            if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
                func_name = node.func.id

                # Check if this function exists in our module
                if func_name in BGYM_FUNCTION_MAP:
                    func = BGYM_FUNCTION_MAP[func_name]

                    # Get function signature to map positional args to parameter names
                    try:
                        sig = inspect.signature(func)
                        param_names = list(sig.parameters.keys())

                        # Process positional arguments
                        for i, arg in enumerate(node.args):
                            if i < len(param_names):
                                param_name = param_names[i]
                                value = _extract_value(arg)
                                start_idx, stop_idx = _get_node_indices(code_string, arg)

                                arg_info = ArgInfo(
                                    function_name=func_name,
                                    name=param_name,
                                    value=value,
                                    type=type(value).__name__,
                                    start_index=start_idx,
                                    stop_index=stop_idx,
                                )
                                result.append(arg_info)

                        # Process keyword arguments
                        for keyword in node.keywords:
                            value = _extract_value(keyword.value)
                            start_idx, stop_idx = _get_node_indices(
                                code_string, keyword.value, keyword
                            )

                            arg_info = ArgInfo(
                                function_name=func_name,
                                name=keyword.arg,
                                value=value,
                                type=type(value).__name__,
                                start_index=start_idx,
                                stop_index=stop_idx,
                            )
                            result.append(arg_info)

                    except Exception as e:
                        # If we can't inspect the function, skip it
                        print(f"Warning: Could not process function {func_name}: {e}")
                        continue

    except SyntaxError as e:
        print(f"Syntax error in code string: {e}")
        return []

    return result

def _extract_value(node: ast.AST) -> Any:
    """
    Extract the actual value from an AST node.

    Args:
        node: AST node representing a value

    Returns:
        The extracted Python value
    """
    if isinstance(node, ast.Constant):
        # Python 3.8+ uses ast.Constant for all literals
        return node.value
    elif isinstance(node, ast.Str):
        # Fallback for older Python versions
        return node.s
    elif isinstance(node, ast.Num):
        # Fallback for older Python versions
        return node.n
    elif isinstance(node, ast.List):
        # Handle list literals
        return [_extract_value(item) for item in node.elts]
    elif isinstance(node, ast.Name):
        # Handle variable names (return as string identifier)
        return node.id
    else:
        # For other node types, return a string representation
        return ast.unparse(node) if hasattr(ast, "unparse") else str(node)

def _get_node_indices(
    source: str, node: ast.AST, keyword_node: ast.keyword = None
) -> tuple[int, int]:
    """
    Convert AST node line/column positions to absolute character indices.

    Args:
        source: Original source code string
        node: AST node (the value)
        keyword_node: If provided, use this keyword node's position as start

    Returns:
        Tuple of (start_index, stop_index) in the source string
    """
    lines = source.splitlines(keepends=True)

    # For keyword arguments, start from the keyword name
    if keyword_node is not None:
        start_line = keyword_node.lineno
        start_col = keyword_node.col_offset
    else:
        start_line = node.lineno
        start_col = node.col_offset

    # Calculate start index
    start_index = 0
    for i in range(start_line - 1):  # lineno is 1-based
        start_index += len(lines[i])
    start_index += start_col

    # End index always comes from the value node
    if hasattr(node, "end_lineno") and hasattr(node, "end_col_offset"):
        end_index = 0
        for i in range(node.end_lineno - 1):
            end_index += len(lines[i])
        end_index += node.end_col_offset
    else:
        # Fallback estimation
        if hasattr(ast, "get_source_segment"):
            segment = ast.get_source_segment(source, node)
            end_index = start_index + len(segment) if segment else start_index + 1
        else:
            end_index = start_index + 1

    return start_index, end_index

def find_bids_and_xy_pairs(args: list[ArgInfo]) -> list[ArgInfo]:
    """
    Find bid arguments and x,y coordinate pairs from a list of ArgInfo objects.

    Args:
        args: List of ArgInfo objects from parse_function_calls

    Returns:
        List of ArgInfo objects containing:
        - Original bid arguments (unchanged)
        - Merged x,y pairs with joint names, tuple values, and combined indices

    Rules for x,y pairs:
    - Must be consecutive arguments
    - Must end with 'x' and 'y' respectively
    - Must have the same prefix (everything before 'x'/'y')
    - Merged name: prefix + "_xy"
    - Merged value: (x_value, y_value) as tuple of floats
    - Merged indices: start of x to stop of y
    """
    result = []
    i = 0

    while i < len(args):
        current_arg = args[i]

        # Check if current arg name ends with 'bid'
        if current_arg.name.endswith("bid"):
            result.append(current_arg)
            i += 1
            continue

        # Check for x,y pair
        if i + 1 < len(args) and current_arg.name.endswith("x") and args[i + 1].name.endswith("y"):

            next_arg = args[i + 1]

            # Extract prefixes (everything before 'x' and 'y')
            current_prefix = current_arg.name[:-1]  # Remove 'x'
            next_prefix = next_arg.name[:-1]  # Remove 'y'

            # Check if they have the same prefix and are from the same function
            if (
                current_prefix == next_prefix
                and current_arg.function_name == next_arg.function_name
            ):

                # Create merged ArgInfo for x,y pair
                merged_name = f"{current_prefix}xy"

                # Convert values to floats and create tuple
                try:
                    x_val = float(current_arg.value)
                    y_val = float(next_arg.value)
                    merged_value = (x_val, y_val)
                except (ValueError, TypeError):
                    # If conversion fails, keep original values
                    merged_value = (current_arg.value, next_arg.value)

                merged_arg = ArgInfo(
                    function_name=current_arg.function_name,
                    name=merged_name,
                    value=merged_value,
                    type="tuple",
                    start_index=current_arg.start_index,
                    stop_index=next_arg.stop_index,
                )

                result.append(merged_arg)
                i += 2  # Skip both x and y args
                continue

        # If no special handling, skip this argument
        i += 1

    return result

def overlay_cross(
    img: Image.Image,
    coord: tuple[float, float],
    color: Union[str, tuple[int, int, int]] = "red",
    length: int = 7,
    width: int = 1,
) -> Image.Image:
    draw = ImageDraw.Draw(img)

    x, y = coord
    half_len = length // 2

    # Draw horizontal line
    draw.line([x - half_len, y, x + half_len, y], fill=color, width=width)
    # Draw vertical line
    draw.line([x, y - half_len, x, y + half_len], fill=color, width=width)

    return img

def overlay_rectangle(
    img: Image.Image,
    bbox: tuple[float, float, float, float],
    color: Union[str, tuple[int, int, int]] = "red",
    width: int = 1,
    dashed: bool = True,
) -> Image.Image:
    draw = ImageDraw.Draw(img)

    x, y, w, h = bbox

    if dashed:
        # Draw dashed rectangle
        linedashed(draw, x, y, x + w, y, color, width)
        linedashed(draw, x + w, y, x + w, y + h, color, width)
        linedashed(draw, x + w, y + h, x, y + h, color, width)
        linedashed(draw, x, y + h, x, y, color, width)
    else:
        draw.rectangle([x, y, x + w, y + h], outline=color, width=width)

    return img

# Adapted from https://stackoverflow.com/questions/51908563/dotted-or-dashed-line-with-python-pillow/58885306#58885306
def linedashed(
    draw: PIL.ImageDraw.Draw, x0, y0, x1, y1, fill, width, dash_length=4, nodash_length=8
):
    line_dx = x1 - x0  # delta x (can be negative)
    line_dy = y1 - y0  # delta y (can be negative)
    line_length = math.hypot(line_dx, line_dy)  # line length (positive)
    if line_length == 0:
        return  # Avoid division by zero in case the line length is 0
    pixel_dx = line_dx / line_length  # x add for 1px line length
    pixel_dy = line_dy / line_length  # y add for 1px line length
    dash_start = 0
    while dash_start < line_length:
        dash_end = dash_start + dash_length
        if dash_end > line_length:
            dash_end = line_length
        draw.line(
            (
                round(x0 + pixel_dx * dash_start),
                round(y0 + pixel_dy * dash_start),
                round(x0 + pixel_dx * dash_end),
                round(y0 + pixel_dy * dash_end),
            ),
            fill=fill,
            width=width,
        )
        dash_start += dash_length + nodash_length

def annotate_action(
    img: Image.Image, action_string: str, properties: dict[str, tuple], colormap: str = "tab10"
) -> str:
    """
    Annotate an image with overlays for action arguments and return colored HTML.

    Args:
        img: PIL Image to modify in place
        action_string: String containing function calls
        properties: Dict mapping bid strings to bounding boxes (x1, y1, x2, y2)
        colormap: Matplotlib colormap name for auto-color selection

    Returns:
        HTML string with arguments colored to match overlays
    """
    # Parse function calls to get all arguments
    all_args = parse_function_calls(action_string)

    # Filter to get bids and xy pairs
    filtered_args = find_bids_and_xy_pairs(all_args)

    # Get colormap
    cmap = plt.get_cmap(colormap)

    # Track colors for each filtered argument
    colors = []

    # Add overlays to image
    for i, arg_info in enumerate(filtered_args):
        # Get color from colormap
        color_rgb = cmap(i % cmap.N)
        color_255 = tuple(int(c * 255) for c in color_rgb[:3])  # Convert to 0-255 range

        colors.append(color_rgb[:3])  # Store normalized RGB for HTML

        if arg_info.name.endswith("xy"):
            # Handle x,y coordinate pairs
            x, y = arg_info.value
            overlay_cross(img, (x, y), color_255, length=9, width=3)

        elif arg_info.name.endswith("bid"):
            # Handle bid arguments with bounding boxes
            bid_value = arg_info.value
            if bid_value in properties:

                bbox = properties[bid_value]["bbox"]
                if bbox:
                    overlay_rectangle(img, bbox, color_255, width=3)

    # Generate colored HTML
    html = create_colored_html(action_string, filtered_args, colors)

    return html

def create_colored_html(action_string: str, filtered_args: list, colors: list) -> str:
    """
    Create HTML with colored arguments using start/stop indices.

    Args:
        action_string: Original action string
        filtered_args: List of ArgInfo objects with start_index/stop_index
        colors: List of RGB tuples, same length as filtered_args

    Returns:
        HTML string with colored spans
    """
    # Sort args by start position for sequential processing
    sorted_pairs = sorted(zip(filtered_args, colors), key=lambda x: x[0].start_index)

    # Build HTML with colored spans
    html_parts = []
    last_end = 0

    for arg_info, color_rgb in sorted_pairs:
        # Add uncolored text before this argument
        html_parts.append(action_string[last_end : arg_info.start_index])

        # Get the argument text
        arg_text = action_string[arg_info.start_index : arg_info.stop_index]

        # Convert color to hex
        color_hex = "#{:02x}{:02x}{:02x}".format(
            int(color_rgb[0] * 255), int(color_rgb[1] * 255), int(color_rgb[2] * 255)
        )

        # Add colored span
        html_parts.append(f'{arg_text}')

        last_end = arg_info.stop_index

    # Add remaining text
    html_parts.append(action_string[last_end:])

    return "".join(html_parts)


ArgInfo

import ast
import inspect
import math
from dataclasses import dataclass
from typing import Any, Union

import matplotlib.pyplot as plt
import PIL
from browsergym.core.action.highlevel import ACTION_SUBSETS
from PIL import Image, ImageDraw

BGYM_FUNCTION_MAP = {}
for subset in ("bid", "coord"):
    for func in ACTION_SUBSETS[subset]:
        if func not in BGYM_FUNCTION_MAP:
            BGYM_FUNCTION_MAP[func.__name__] = func

@dataclass
class ArgInfo:
    function_name: str
    name: str
    value: Any
    type: str
    start_index: int
    stop_index: int

def parse_function_calls(code_string: str) -> list[ArgInfo]:
    """
    Parse a string containing multiple function calls and return a list of ArgInfo objects
    for all arguments in all function calls.

    Args:
        code_string: String containing function calls

    Returns:
        List of ArgInfo objects containing detailed information about each argument

    Example:
        >>> code = '''
        ... mouse_click(34, 59)
        ... fill("a234", "test")
        ... '''
        >>> result = parse_function_calls(code)
        >>> # Returns list of ArgInfo objects for each argument


from browsergym.core.action.highlevel import ACTION_SUBSETS
from PIL import Image, ImageDraw

BGYM_FUNCTION_MAP = {}
for subset in ("bid", "coord"):
    for func in ACTION_SUBSETS[subset]:
        if func not in BGYM_FUNCTION_MAP:
            BGYM_FUNCTION_MAP[func.__name__] = func

@dataclass
class ArgInfo:
    function_name: str
    name: str
    value: Any
    type: str
    start_index: int
    stop_index: int

def parse_function_calls(code_string: str) -> list[ArgInfo]:
    """
    Parse a string containing multiple function calls and return a list of ArgInfo objects
    for all arguments in all function calls.

    Args:
        code_string: String containing function calls

    Returns:
        List of ArgInfo objects containing detailed information about each argument

    Example:
        >>> code = '''
        ... mouse_click(34, 59)
        ... fill("a234", "test")
        ... '''
        >>> result = parse_function_calls(code)
        >>> # Returns list of ArgInfo objects for each argument
    """
    result = []

    try:
        # Parse the code string into an AST
        tree = ast.parse(code_string)

        # Extract all function calls
        for node in ast.walk(tree):
            if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
                func_name = node.func.id

                # Check if this function exists in our module
                if func_name in BGYM_FUNCTION_MAP:
                    func = BGYM_FUNCTION_MAP[func_name]

                    # Get function signature to map positional args to parameter names
                    try:
                        sig = inspect.signature(func)
                        param_names = list(sig.parameters.keys())

                        # Process positional arguments
                        for i, arg in enumerate(node.args):
                            if i < len(param_names):
                                param_name = param_names[i]
                                value = _extract_value(arg)
                                start_idx, stop_idx = _get_node_indices(code_string, arg)

                                arg_info = ArgInfo(
                                    function_name=func_name,
                                    name=param_name,
                                    value=value,
                                    type=type(value).__name__,
                                    start_index=start_idx,
                                    stop_index=stop_idx,
                                )
                                result.append(arg_info)

                        # Process keyword arguments
                        for keyword in node.keywords:
                            value = _extract_value(keyword.value)
                            start_idx, stop_idx = _get_node_indices(
                                code_string, keyword.value, keyword
                            )

                            arg_info = ArgInfo(
                                function_name=func_name,
                                name=keyword.arg,
                                value=value,
                                type=type(value).__name__,
                                start_index=start_idx,
                                stop_index=stop_idx,
                            )
                            result.append(arg_info)

                    except Exception as e:
                        # If we can't inspect the function, skip it
                        print(f"Warning: Could not process function {func_name}: {e}")
                        continue

    except SyntaxError as e:
        print(f"Syntax error in code string: {e}")
        return []

    return result

def _extract_value(node: ast.AST) -> Any:
    """
    Extract the actual value from an AST node.

    Args:
        node: AST node representing a value

    Returns:
        The extracted Python value
    """
    if isinstance(node, ast.Constant):
        # Python 3.8+ uses ast.Constant for all literals
        return node.value
    elif isinstance(node, ast.Str):
        # Fallback for older Python versions
        return node.s
    elif isinstance(node, ast.Num):
        # Fallback for older Python versions


                                name=keyword.arg,
                                value=value,
                                type=type(value).__name__,
                                start_index=start_idx,
                                stop_index=stop_idx,
                            )
                            result.append(arg_info)

                    except Exception as e:
                        # If we can't inspect the function, skip it
                        print(f"Warning: Could not process function {func_name}: {e}")
                        continue

    except SyntaxError as e:
        print(f"Syntax error in code string: {e}")
        return []

    return result

def _extract_value(node: ast.AST) -> Any:
    """
    Extract the actual value from an AST node.

    Args:
        node: AST node representing a value

    Returns:
        The extracted Python value
    """
    if isinstance(node, ast.Constant):
        # Python 3.8+ uses ast.Constant for all literals
        return node.value
    elif isinstance(node, ast.Str):
        # Fallback for older Python versions
        return node.s
    elif isinstance(node, ast.Num):
        # Fallback for older Python versions
        return node.n
    elif isinstance(node, ast.List):
        # Handle list literals
        return [_extract_value(item) for item in node.elts]
    elif isinstance(node, ast.Name):
        # Handle variable names (return as string identifier)
        return node.id
    else:
        # For other node types, return a string representation
        return ast.unparse(node) if hasattr(ast, "unparse") else str(node)

def _get_node_indices(
    source: str, node: ast.AST, keyword_node: ast.keyword = None
) -> tuple[int, int]:
    """
    Convert AST node line/column positions to absolute character indices.

    Args:
        source: Original source code string
        node: AST node (the value)
        keyword_node: If provided, use this keyword node's position as start

    Returns:
        Tuple of (start_index, stop_index) in the source string
    """
    lines = source.splitlines(keepends=True)

    # For keyword arguments, start from the keyword name
    if keyword_node is not None:


    if isinstance(node, ast.Constant):
        # Python 3.8+ uses ast.Constant for all literals
        return node.value
    elif isinstance(node, ast.Str):
        # Fallback for older Python versions
        return node.s
    elif isinstance(node, ast.Num):
        # Fallback for older Python versions
        return node.n
    elif isinstance(node, ast.List):
        # Handle list literals
        return [_extract_value(item) for item in node.elts]
    elif isinstance(node, ast.Name):
        # Handle variable names (return as string identifier)
        return node.id
    else:
        # For other node types, return a string representation
        return ast.unparse(node) if hasattr(ast, "unparse") else str(node)

def _get_node_indices(
    source: str, node: ast.AST, keyword_node: ast.keyword = None
) -> tuple[int, int]:
    """
    Convert AST node line/column positions to absolute character indices.

    Args:
        source: Original source code string
        node: AST node (the value)
        keyword_node: If provided, use this keyword node's position as start

    Returns:
        Tuple of (start_index, stop_index) in the source string
    """
    lines = source.splitlines(keepends=True)

    # For keyword arguments, start from the keyword name
    if keyword_node is not None:
        start_line = keyword_node.lineno
        start_col = keyword_node.col_offset
    else:
        start_line = node.lineno
        start_col = node.col_offset

    # Calculate start index
    start_index = 0
    for i in range(start_line - 1):  # lineno is 1-based
        start_index += len(lines[i])
    start_index += start_col

    # End index always comes from the value node
    if hasattr(node, "end_lineno") and hasattr(node, "end_col_offset"):
        end_index = 0
        for i in range(node.end_lineno - 1):
            end_index += len(lines[i])
        end_index += node.end_col_offset
    else:
        # Fallback estimation
        if hasattr(ast, "get_source_segment"):
            segment = ast.get_source_segment(source, node)
            end_index = start_index + len(segment) if segment else start_index + 1
        else:
            end_index = start_index + 1

    return start_index, end_index

def find_bids_and_xy_pairs(args: list[ArgInfo]) -> list[ArgInfo]:
    """
    Find bid arguments and x,y coordinate pairs from a list of ArgInfo objects.

    Args:
        args: List of ArgInfo objects from parse_function_calls

    Returns:
        List of ArgInfo objects containing:
        - Original bid arguments (unchanged)
        - Merged x,y pairs with joint names, tuple values, and combined indices

    Rules for x,y pairs:
    - Must be consecutive arguments
    - Must end with 'x' and 'y' respectively
    - Must have the same prefix (everything before 'x'/'y')
    - Merged name: prefix + "_xy"
    - Merged value: (x_value, y_value) as tuple of floats