TapeAgent

    def make_agent(self) -> bgym.Agent:
        agent: Agent = hydra.utils.instantiate(self.config.agent)
        return TapeAgent(agent=agent)

@dataclass
class TapeAgentInfo(bgym.AgentInfo):
    thoughts: list[Thought] = None  # type: ignore

class DictObservation(Observation):
    """
    Container for wrapping old dict observation into new Observation class.
    """

    kind: Literal["dict_observation"] = "dict_observation"  # type: ignore
    content: str

class TapeAgent(bgym.Agent):
    agent: Agent
    tape: Tape

    def __init__(self, agent: Agent):
        super().__init__()
        self.agent = agent
        self.tape = Tape(steps=[])

    def obs_preprocessor(self, obs: Observation | list[Observation]) -> list[Observation]:
        if isinstance(obs, Observation):
            obs = [obs]
        assert isinstance(obs, list), f"Expected list of Observations, got {type(obs)}"
        logger.info(f"Observations: {[type(o).__name__ for o in obs]}")
        return obs

    def get_action(self, obs: Observation | list[Observation]) -> tuple[Action, TapeAgentInfo]:
        self.tape += obs  # type: ignore
        thoughts: list[Thought] = []
        action = None
        while not action:
            for event in self.agent.run(self.tape):
                if not event.step:
                    continue
                self.tape = self.tape.append(event.step)
                if isinstance(event.step, Thought):
                    thoughts.append(event.step)
                    logger.info(f"Thought: {event.step.llm_view()}")
                elif isinstance(event.step, Action) and not action:  # we use first action only
                    action = event.step
                    logger.info(f"Action: {action.llm_view()}")
                else:
                    # there could be control flow steps for switching nodes and if clauses
                    logger.info(f"Other step: {type(event.step)}")
        logger.info(f"Tape after run: ({len(self.tape)}) {[type(s).__name__ for s in self.tape]}")
        return (action, TapeAgentInfo(thoughts=thoughts))

    @property
    def final_tape(self) -> Tape:
        truncated = not any([isinstance(s, StopStep) for s in self.tape.steps])
        self.tape.metadata = ExtendedMetadata(author=self.agent.name, truncated=truncated)
        return self.tape


    truncated: bool = False
    reward: float = 0.0
    attempt_number: int = 0
    other: dict = {}

class Tape(BaseTape):
    metadata: ExtendedMetadata = Field(default_factory=ExtendedMetadata)  # type: ignore

def load_config(config_name: str) -> DictConfig:
    with hydra.initialize(config_path="conf", version_base="1.1"):
        config = hydra.compose(config_name=config_name)
    return config

@dataclass
class TapeAgentArgs(AgentArgs):
    config: DictConfig = None  # type: ignore

    def make_agent(self) -> bgym.Agent:
        agent: Agent = hydra.utils.instantiate(self.config.agent)
        return TapeAgent(agent=agent)

@dataclass
class TapeAgentInfo(bgym.AgentInfo):
    thoughts: list[Thought] = None  # type: ignore

class DictObservation(Observation):
    """
    Container for wrapping old dict observation into new Observation class.
    """

    kind: Literal["dict_observation"] = "dict_observation"  # type: ignore
    content: str

class TapeAgent(bgym.Agent):
    agent: Agent
    tape: Tape


@dataclass
class TapeAgentInfo(bgym.AgentInfo):
    thoughts: list[Thought] = None  # type: ignore

class DictObservation(Observation):
    """
    Container for wrapping old dict observation into new Observation class.
    """

    kind: Literal["dict_observation"] = "dict_observation"  # type: ignore
    content: str

class TapeAgent(bgym.Agent):
    agent: Agent
    tape: Tape

    def __init__(self, agent: Agent):
        super().__init__()
        self.agent = agent
        self.tape = Tape(steps=[])

    def obs_preprocessor(self, obs: Observation | list[Observation]) -> list[Observation]:
        if isinstance(obs, Observation):
            obs = [obs]
        assert isinstance(obs, list), f"Expected list of Observations, got {type(obs)}"
        logger.info(f"Observations: {[type(o).__name__ for o in obs]}")
        return obs

    def get_action(self, obs: Observation | list[Observation]) -> tuple[Action, TapeAgentInfo]:
        self.tape += obs  # type: ignore
        thoughts: list[Thought] = []
        action = None
        while not action:
            for event in self.agent.run(self.tape):
                if not event.step:
                    continue
                self.tape = self.tape.append(event.step)
                if isinstance(event.step, Thought):
                    thoughts.append(event.step)
                    logger.info(f"Thought: {event.step.llm_view()}")


class DictObservation(Observation):
    """
    Container for wrapping old dict observation into new Observation class.
    """

    kind: Literal["dict_observation"] = "dict_observation"  # type: ignore
    content: str

class TapeAgent(bgym.Agent):
    agent: Agent
    tape: Tape

    def __init__(self, agent: Agent):
        super().__init__()
        self.agent = agent
        self.tape = Tape(steps=[])

    def obs_preprocessor(self, obs: Observation | list[Observation]) -> list[Observation]:
        if isinstance(obs, Observation):
            obs = [obs]
        assert isinstance(obs, list), f"Expected list of Observations, got {type(obs)}"
        logger.info(f"Observations: {[type(o).__name__ for o in obs]}")
        return obs

    def get_action(self, obs: Observation | list[Observation]) -> tuple[Action, TapeAgentInfo]:
        self.tape += obs  # type: ignore
        thoughts: list[Thought] = []
        action = None
        while not action:
            for event in self.agent.run(self.tape):
                if not event.step:
                    continue
                self.tape = self.tape.append(event.step)
                if isinstance(event.step, Thought):
                    thoughts.append(event.step)
                    logger.info(f"Thought: {event.step.llm_view()}")
                elif isinstance(event.step, Action) and not action:  # we use first action only
                    action = event.step
                    logger.info(f"Action: {action.llm_view()}")
                else:
                    # there could be control flow steps for switching nodes and if clauses
                    logger.info(f"Other step: {type(event.step)}")
        logger.info(f"Tape after run: ({len(self.tape)}) {[type(s).__name__ for s in self.tape]}")


    kind: Literal["dict_observation"] = "dict_observation"  # type: ignore
    content: str

class TapeAgent(bgym.Agent):
    agent: Agent
    tape: Tape

    def __init__(self, agent: Agent):
        super().__init__()
        self.agent = agent
        self.tape = Tape(steps=[])

    def obs_preprocessor(self, obs: Observation | list[Observation]) -> list[Observation]:
        if isinstance(obs, Observation):
            obs = [obs]
        assert isinstance(obs, list), f"Expected list of Observations, got {type(obs)}"
        logger.info(f"Observations: {[type(o).__name__ for o in obs]}")
        return obs

    def get_action(self, obs: Observation | list[Observation]) -> tuple[Action, TapeAgentInfo]:
        self.tape += obs  # type: ignore
        thoughts: list[Thought] = []
        action = None
        while not action:
            for event in self.agent.run(self.tape):
                if not event.step:
                    continue
                self.tape = self.tape.append(event.step)
                if isinstance(event.step, Thought):
                    thoughts.append(event.step)
                    logger.info(f"Thought: {event.step.llm_view()}")
                elif isinstance(event.step, Action) and not action:  # we use first action only
                    action = event.step
                    logger.info(f"Action: {action.llm_view()}")
                else:
                    # there could be control flow steps for switching nodes and if clauses
                    logger.info(f"Other step: {type(event.step)}")
        logger.info(f"Tape after run: ({len(self.tape)}) {[type(s).__name__ for s in self.tape]}")
        return (action, TapeAgentInfo(thoughts=thoughts))

    @property
    def final_tape(self) -> Tape:
        truncated = not any([isinstance(s, StopStep) for s in self.tape.steps])
        self.tape.metadata = ExtendedMetadata(author=self.agent.name, truncated=truncated)
        return self.tape


        thoughts: list[Thought] = []
        action = None
        while not action:
            for event in self.agent.run(self.tape):
                if not event.step:
                    continue
                self.tape = self.tape.append(event.step)
                if isinstance(event.step, Thought):
                    thoughts.append(event.step)
                    logger.info(f"Thought: {event.step.llm_view()}")
                elif isinstance(event.step, Action) and not action:  # we use first action only
                    action = event.step
                    logger.info(f"Action: {action.llm_view()}")
                else:
                    # there could be control flow steps for switching nodes and if clauses
                    logger.info(f"Other step: {type(event.step)}")
        logger.info(f"Tape after run: ({len(self.tape)}) {[type(s).__name__ for s in self.tape]}")
        return (action, TapeAgentInfo(thoughts=thoughts))

    @property
    def final_tape(self) -> Tape:
        truncated = not any([isinstance(s, StopStep) for s in self.tape.steps])
        self.tape.metadata = ExtendedMetadata(author=self.agent.name, truncated=truncated)
        return self.tape


import logging
import os

from agentlab.agents.tapeagent.agent import TapeAgentArgs, load_config
from agentlab.benchmarks.gaia import GaiaBenchmark, stop_old_sandbox
from agentlab.experiments.study import make_study

fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

if __name__ == "__main__":
    config = load_config("gaia_l1")
    study = make_study(
        benchmark=GaiaBenchmark.from_config(config),  # type: ignore
        agent_args=TapeAgentArgs(agent_name=config.name, config=config),
        comment=config.comment,
        logging_level=logging.INFO,
        logging_level_stdout=logging.INFO,
    )
    stop_old_sandbox()
    if os.environ.get("AGENTLAB_DEBUG"):
        study.exp_args_list = study.exp_args_list[:3]
        study.run(n_jobs=1, n_relaunch=1, parallel_backend="sequential")
    else:
        study.run(n_jobs=config.n_jobs, n_relaunch=1, parallel_backend=config.parallel_backend)


main.py

"""
Note: This script is a convenience script to launch experiments instead of using
the command line.

Copy this script and modify at will, but don't push your changes to the
repository.
"""

import logging

from agentlab.agents.generic_agent import (
    AGENT_LLAMA3_70B,
    AGENT_LLAMA31_70B,
    RANDOM_SEARCH_AGENT,
    AGENT_4o,
    AGENT_4o_MINI,
    AGENT_o3_MINI,
    AGENT_37_SONNET,
    AGENT_CLAUDE_SONNET_35,
    AGENT_GPT5_MINI,
)


"""
Note: This script is a convenience script to launch experiments instead of using
the command line.

Copy this script and modify at will, but don't push your changes to the
repository.
"""

import logging
from copy import deepcopy

import bgym

from agentlab.agents.tool_use_agent.tool_use_agent import (
    DEFAULT_PROMPT_CONFIG,
    GPT_4_1,
    ToolUseAgentArgs,
)
from agentlab.experiments.study import Study

logging.getLogger().setLevel(logging.INFO)


import os
from pathlib import Path
from agentlab.experiments.study import Study

base_dir = "/home/toolkit/ui_copilot_results"

exp_paths = [
    "2025-01-31_22-08-34_genericagent-o3-mini-2025-01-31-on-workarena-l1",
    #  '2025-02-02_01-53-45_genericagent-openai-o1-mini-2024-09-12-on-workarena-l1',
    "2025-02-02_01-55-04_genericagent-openai-o1-mini-2024-09-12-on-workarena-l1",
]
full_paths = [os.path.join(base_dir, exp_path) for exp_path in exp_paths]

for full_path in full_paths:
    study = Study.load(Path(full_path))

    study.append_to_journal(strict_reproducibility=False)


import subprocess
import sys
from pathlib import Path

import pytest

@pytest.mark.pricy
def test_main_script_execution():
    # this should trigger agent_4o_mini on miniwob_tiny_test unless this was
    # reconfigured differently.
    path = Path(__file__).parent.parent / "main.py"

    sys.path.insert(0, str(path.parent))

    # just make sure it's in the right state
    main = __import__(path.stem)
    assert main.benchmark == "miniwob_tiny_test"
    assert main.reproducibility_mode == False
    assert main.relaunch == False
    assert main.n_jobs <= 10


from agentlab.ui_assistant import make_exp_args
from agentlab.agents.generic_agent import AGENT_4o

def test_make_exp_args():
    """Basic unit test to detect refactoring errors."""
    exp_args = make_exp_args(AGENT_4o, "https://www.google.com")

    assert exp_args.agent_args.flags.action.demo_mode == "default"


import os
import time
from concurrent.futures import ThreadPoolExecutor, as_completed

import anthropic

client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])

def make_request(messages):
    response = client.messages.create(
        model="claude-3-5-sonnet-20241022", max_tokens=10, messages=messages
    )
    return response.usage

def make_message(text):
    return {
        "role": "user",
        "content": [
            {


from PIL import Image

from agentlab.analyze import overlay_utils

def test_parse_function_calls():

    test_code = """
mouse_click(34, 59)
fill("a234", "test")
click('b123', button="right", modifiers=["Shift", "Control"])
select_option("c456", ["option1", "option2"])
"""

    result = overlay_utils.parse_function_calls(test_code)

    assert result[1].function_name == "mouse_click"
    assert result[1].name == "y"
    assert test_code[result[1].start_index : result[1].stop_index] == "59"

    assert result[8].function_name == "select_option"


from pathlib import Path
import shutil
import tempfile

import pandas as pd
from agentlab.analyze.inspect_results import get_study_summary

def test_get_study_summary():

    with tempfile.TemporaryDirectory() as tmp_dir:
        study_dir = Path(tmp_dir) / "test_study"

        study_dir_original = Path(__file__).parent.parent / "data" / "test_study"

        # recursively copy the study to the temp dir using shutil
        shutil.copytree(study_dir_original, study_dir)

        sentinel = {}

        summary = get_study_summary(study_dir, sentinel=sentinel)


import pytest

from agentlab.llm.chat_api import HuggingFaceURLChatModel, make_system_message, make_user_message
from agentlab.llm.llm_utils import download_and_save_model
from agentlab.llm.prompt_templates import STARCHAT_PROMPT_TEMPLATE

# TODO(optimass): figure out a good model for all tests

@pytest.mark.skip(reason="Requires a local model checkpoint")
def test_CustomLLMChatbot_locally():
    # model_path = "google/flan-t5-base"  # remote model on HuggingFace Hub
    model_path = "/mnt/ui_copilot/data_rw/models/starcoderbase-1b-ft"  # local model in shared volum

    chatbot = HuggingFaceURLChatModel(model_path=model_path, temperature=1e-3)

    messages = [
        make_system_message("Please tell me back the following word: "),
        make_user_message("bird"),
    ]


from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT
from agentlab.llm.chat_api import BaseModelArgs

def test_llm_configs():

    for _, args in CHAT_MODEL_ARGS_DICT.items():
        assert isinstance(args, BaseModelArgs)


import os
import time
from functools import partial

import pytest

import agentlab.llm.tracking as tracking
from agentlab.llm.chat_api import (
    AzureChatModel,
    OpenAIChatModel,
    OpenRouterChatModel,
    make_system_message,
    make_user_message,
)

def test_get_action_decorator():
    action, agent_info = tracking.cost_tracker_decorator(lambda x, y: call_llm())(None, None)
    assert action == "action"
    assert agent_info["stats"] == {
        "input_tokens": 1,


import os
from functools import partial

import pytest
from agentlab.llm.litellm_api import LiteLLMModelArgs
from agentlab.llm.response_api import APIPayload, LLMOutput

chat_api_tools = [
    {
        "type": "function",
        "name": "get_weather",
        "description": "Get the current weather in a given location.",
        "parameters": {
            "type": "object",
            "properties": {
                "location": {
                    "type": "string",
                    "description": "The location to get the weather for.",
                },
                "unit": {
                    "type": "string",


import os
from typing import Any, Dict, List, Optional
from unittest.mock import MagicMock, patch

import anthropic
import openai
import pytest

from agentlab.llm import tracking
from agentlab.llm.response_api import (
    AnthropicAPIMessageBuilder,
    APIPayload,
    ClaudeResponseModelArgs,
    LLMOutput,
    OpenAIChatCompletionAPIMessageBuilder,
    OpenAIChatModelArgs,
    OpenAIResponseAPIMessageBuilder,
    OpenAIResponseModelArgs,
)


import os

import pytest

from agentlab.llm.chat_api import (
    AnthropicModelArgs,
    AzureModelArgs,
    OpenAIModelArgs,
    make_system_message,
    make_user_message,
)

# TODO(optimass): figure out a good model for all tests

if "AGENTLAB_LOCAL_TEST" in os.environ:
    skip_tests = os.environ["AGENTLAB_LOCAL_TEST"] != "1"
else:
    skip_tests = False


import warnings
from typing import Literal
from unittest.mock import Mock

import httpx
import pytest
from openai import RateLimitError

from agentlab.llm import llm_utils
from agentlab.llm.chat_api import make_system_message

yaml_str = """Analysis:
This is the analysis

Summary: This is the summary

Confidence Score: 7
"""

def test_yaml_parser():


import math
import tempfile
from pathlib import Path

import pytest

from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_3_5, AGENT_4o_MINI
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.analyze import inspect_results
from agentlab.experiments.launch_exp import (
    find_incomplete,
    non_dummy_count,
    run_experiments,
)
from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.experiments.study import Study
from agentlab.llm.chat_api import CheatMiniWoBLLMArgs

def test_relaunch_study():
    study_dir = Path(__file__).parent.parent / "data" / "test_study"


from agentlab.experiments.multi_server import WebArenaInstanceVars
from browsergym.webarena.instance import WebArenaInstance

def test_webarena_multiserver():

    instance_1 = WebArenaInstanceVars(
        base_url="http://webarena1.eastus.cloudapp.azure.com",
        shopping="8082/",
        shopping_admin="8083/admin",
        reddit="8080",
        gitlab="9001",
        wikipedia="8081/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing",
        map="443",
        homepage="80",
        full_reset="7565",
        module_name="webarena",
        prefix="WA_",
    )

    instance_1.init()


import bgym
import pytest
import ray
from flaky import flaky

from agentlab.experiments.exp_utils import MockedExpArgs, add_dependencies
from agentlab.experiments.graph_execution_ray import execute_task_graph

TASK_TIME = 3

@flaky(max_runs=3, min_passes=1)
def test_execute_task_graph():
    # Define a list of ExpArgs with dependencies
    exp_args_list = [
        MockedExpArgs(exp_id="task1", depends_on=[]),
        MockedExpArgs(exp_id="task2", depends_on=["task1"]),
        MockedExpArgs(exp_id="task3", depends_on=["task1"]),
        MockedExpArgs(exp_id="task4", depends_on=["task2", "task3"]),
    ]


from ast import mod
from dataclasses import dataclass
from agentlab.experiments.args import (
    expand_cross_product,
    CrossProd,
    Choice,
    make_progression_study,
    sample_args,
    make_ablation_study,
)

@dataclass
class LLMArgsTest:
    model_name: str = "model1"
    temperature: float = 0.1

@dataclass
class ExpArgsTest:
    llm_args: LLMArgsTest


import json
import tempfile
import time
from pathlib import Path

import bgym
import pytest
from bgym import DEFAULT_BENCHMARKS

from agentlab.agents.generic_agent import AGENT_4o_MINI
from agentlab.analyze import inspect_results
from agentlab.experiments import reproducibility_util

@pytest.mark.parametrize(
    "benchmark_name",
    ["miniwob", "workarena_l1", "webarena", "visualwebarena"],
)
def test_get_reproducibility_info(benchmark_name):

    benchmark = DEFAULT_BENCHMARKS[benchmark_name]()


import pytest
from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_4o
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.llm.chat_api import CheatMiniWoBLLMArgs
from agentlab.experiments.study import ParallelStudies, make_study, Study
from agentlab.experiments.multi_server import WebArenaInstanceVars
import logging

logging.getLogger().setLevel(logging.INFO)

def _make_agent_args_list():
    # CheatMiniWoB agents won't succeed on WebArena, this is just for testing parallelization
    agent_args_list = []
    for i in range(2):
        agent_args = GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs(),
            flags=FLAGS_GPT_4o,
        )


import importlib.util
import tempfile
from pathlib import Path
from unittest.mock import patch

import pytest

spec = importlib.util.find_spec("desktop_env")
if spec is None:
    DESKTOP_ENV_AVAILABLE = False
    OSWorldActionSet = None
    OsworldEnvArgs = None
    OsworldGym = None
else:
    # If desktop_env is available, import the necessary classes
    from agentlab.benchmarks.osworld import (
        OSWorldActionSet,
        OsworldEnvArgs,
        OsworldGym,
    )


import re
import tempfile
from dataclasses import dataclass
from pathlib import Path

from openai import OpenAIError

from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_3_5
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.analyze import inspect_results
from agentlab.experiments import launch_exp
from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.llm.chat_api import BaseModelArgs, CheatMiniWoBLLMArgs
from agentlab.llm.llm_utils import Discussion

def test_generic_agent():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs(),
            flags=FLAGS_GPT_3_5,


import logging
import tempfile

import pytest

from agentlab.agents.visualwebarena.agent import VisualWebArenaAgentArgs
from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT

@pytest.mark.pricy
def test_agent():
    with tempfile.TemporaryDirectory() as exp_dir:
        env_args = EnvArgs(
            task_name="miniwob.click-button",
            task_seed=0,
            max_steps=10,
            headless=True,
        )

        chat_model_args = CHAT_MODEL_ARGS_DICT["openai/gpt-4o-mini-2024-07-18"]


import os
import uuid
from pathlib import Path

try:
    from tapeagents.steps import ImageObservation

    from agentlab.agents.tapeagent.agent import TapeAgent, TapeAgentArgs, load_config
    from agentlab.benchmarks.gaia import GaiaBenchmark, GaiaQuestion
except ModuleNotFoundError:
    import pytest

    pytest.skip("Skipping test due to missing dependencies", allow_module_level=True)

def mock_dataset() -> dict:
    """Mock dataset for testing purposes."""
    data = [{"task_id": str(uuid.uuid4()), "file_name": "", "file_path": ""} for i in range(165)]
    data[5] = {
        "task_id": "32102e3e-d12a-4209-9163-7b3a104efe5d",
        "Question": """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet.""",


from copy import deepcopy

import bgym
import pytest
from bgym import HighLevelActionSet, HighLevelActionSetArgs

from agentlab.agents import dynamic_prompting as dp
from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_3_5
from agentlab.agents.generic_agent.generic_agent_prompt import GenericPromptFlags, MainPrompt
from agentlab.llm.llm_utils import count_tokens

html_template = """
<html>
<body>
<div>
Hello World.
Step {}.
</div>
</body>
some extra text to make the html longer
</html>


# Configuration file for the Sphinx documentation builder.
#
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information

import os
import subprocess

# Automatically retrieve the project version from Git
def get_version():
    try:
        return subprocess.check_output(["git", "describe", "--tags"], encoding="utf-8").strip()
    except Exception:
        return "0.0.0"

project = "AgentLab"


from pathlib import Path

from bgym import DEFAULT_BENCHMARKS
from dotenv import load_dotenv

from agentlab.agents.generic_agent.tmlr_config import (
    BASE_FLAGS,
    CHAT_MODEL_ARGS_DICT,
    GenericAgentArgs,
)
from agentlab.benchmarks.setup_benchmark import ensure_benchmark
from agentlab.experiments.study import Study

# This ensures MiniWob assets are downloaded and sets the MINIWOB_URL .env in the project dir.
project_dir = Path(__file__).parents[2]
ensure_benchmark("miniwob", project_root=project_dir)
load_dotenv(project_dir.joinpath(".env"), override=False)  # load .env variables

agent_config = GenericAgentArgs(
    chat_model_args=CHAT_MODEL_ARGS_DICT[


import json
import logging
import os

from agentlab.agents.tool_use_agent.tool_use_agent import OSWORLD_CLAUDE
from agentlab.benchmarks.osworld import OsworldBenchmark
from agentlab.experiments.study import Study, make_study

fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

def get_most_recent_incomplete_study() -> Study:
    """
    Relaunch an existing study, this will continue incomplete experiments and relaunch errored experiments.
    """
    study = Study.load_most_recent()
    study.find_incomplete(include_errors=True)
    return study


import argparse

from agentlab.agents.agent_args import AgentArgs
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.launch_exp import import_object
from agentlab.experiments.loop import EnvArgs, ExpArgs

def make_exp_args(agent_args: AgentArgs, start_url: str) -> ExpArgs:
    try:
        agent_args.flags.action.demo_mode = "default"
    except AttributeError:
        pass

    if isinstance(agent_args, GenericAgentArgs):
        agent_args.flags.enable_chat = True

    exp_args = ExpArgs(
        agent_args=agent_args,
        env_args=EnvArgs(


import os
from dataclasses import dataclass
from pathlib import Path

import pandas as pd
from tqdm import tqdm

from agentlab.analyze import inspect_results
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.study import Study

@dataclass
class StudyInfo:
    study_dir: Path
    study: Study
    summary_df: pd.DataFrame
    should_delete: bool = False
    reason: str = ""


import ast
import inspect
import math
from dataclasses import dataclass
from typing import Any, Union

import matplotlib.pyplot as plt
import PIL
from browsergym.core.action.highlevel import ACTION_SUBSETS
from PIL import Image, ImageDraw

BGYM_FUNCTION_MAP = {}
for subset in ("bid", "coord"):
    for func in ACTION_SUBSETS[subset]:
        if func not in BGYM_FUNCTION_MAP:
            BGYM_FUNCTION_MAP[func.__name__] = func

@dataclass
class ArgInfo:
    function_name: str


import base64
from io import BytesIO
from pathlib import Path

from agentlab.experiments.loop import ExpResult
from agentlab.experiments.study import get_most_recent_study
from agentlab.llm.llm_utils import BaseMessage as AgentLabBaseMessage

def exp_result_to_html(
    exp_result: ExpResult,
    steps_open: bool = True,
    som_open: bool = False,
    axtree_open: bool = False,
    html_open: bool = False,
    prompt_open: bool = False,
    embed_images: bool = True,
) -> str:
    """
    Convert an ExpResult to HTML with collapsible sections.


import fnmatch
import json
import random
import re
import traceback
import warnings
from collections import defaultdict
from logging import warn
from pathlib import Path

import numpy as np
import pandas as pd
from IPython.display import display
from tqdm import tqdm

from agentlab.experiments.loop import ExpResult, get_exp_result, yield_all_exp_results

# TODO find a more portable way to code set_task_category_as_index at least
# handle dynamic imports. We don't want to always import workarena
# from browsergym.workarena import TASK_CATEGORY_MAP


import json
import logging
import sys
from collections import defaultdict
from pathlib import Path

import numpy as np
import yaml
from tapeagents.core import Step, StepMetadata
from tapeagents.observe import retrieve_all_llm_calls
from tapeagents.renderers.camera_ready_renderer import CameraReadyRenderer
from tapeagents.tape_browser import TapeBrowser

from agentlab.agents.tapeagent.agent import ExtendedMetadata, Tape
from agentlab.benchmarks.gaia import step_error

logger = logging.getLogger(__name__)
fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])


import base64
import html
import os
import traceback
from copy import deepcopy
from io import BytesIO
from logging import warning
from pathlib import Path

import gradio as gr
import matplotlib.patches as patches
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from attr import dataclass
from browsergym.experiments.loop import StepInfo as BGymStepInfo
from langchain.schema import BaseMessage, HumanMessage
from openai import OpenAI
from openai.types.responses import ResponseFunctionToolCall
from PIL import Image


import logging
import os
import time
from functools import partial
from typing import Any, List, Optional, Union

from pydantic import Field

from agentlab.llm.base_api import AbstractChatModel
from agentlab.llm.llm_utils import AIMessage, Discussion
from agentlab.llm.prompt_templates import PromptTemplate, get_prompt_template

class HFBaseChatModel(AbstractChatModel):
    """
    Custom LLM Chatbot that can interface with HuggingFace models with support for multiple samples.

    This class allows for the creation of a custom chatbot using models hosted
    on HuggingFace Hub or a local checkpoint. It provides flexibility in defining
    the temperature for response sampling and the maximum number of new tokens
    in the response.


import json
import logging
import os
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Dict, List, Literal, Optional, Union

import openai
from anthropic import Anthropic
from anthropic.types import Completion
from anthropic.types import Message as AnthrophicMessage
from openai import OpenAI

from agentlab.llm.llm_utils import image_to_png_base64_url

from .base_api import BaseModelArgs
from .llm_utils import (
    call_anthropic_api_with_retries,
    call_openai_api_with_retries,
)
from .tracking import TrackAPIPricingMixin


from abc import ABC, abstractmethod
from dataclasses import dataclass

class AbstractChatModel(ABC):
    @abstractmethod
    def __call__(self, messages: list[dict]) -> dict:
        pass

    def get_stats(self):
        return {}

@dataclass
class BaseModelArgs(ABC):
    """Base class for all model arguments."""

    model_name: str
    max_total_tokens: int = None
    max_input_tokens: int = None
    max_new_tokens: int = None


import json
import logging
from dataclasses import dataclass
from functools import partial
from typing import Any, Dict, List, Optional, Type

import litellm
from litellm import completion
from openai.types.chat import ChatCompletion as OpenAIChatCompletion

from agentlab.llm.base_api import BaseModelArgs
from agentlab.llm.response_api import (
    AgentlabAction,
    APIPayload,
    BaseModelWithPricing,
    LLMOutput,
    Message,
    MessageBuilder,
    OpenAIChatCompletionAPIMessageBuilder,
    ToolCall,
    ToolCalls,


import logging
import os
import re
import time
from dataclasses import dataclass
from functools import partial
from typing import Optional

import anthropic
import openai
from openai import NOT_GIVEN, OpenAI

import agentlab.llm.tracking as tracking
from agentlab.llm.base_api import AbstractChatModel, BaseModelArgs
from agentlab.llm.llm_utils import AIMessage, Discussion

def make_system_message(content: str) -> dict:
    return dict(role="system", content=content)


from openai import NOT_GIVEN

from agentlab.llm.chat_api import (
    AnthropicModelArgs,
    AzureModelArgs,
    OpenAIModelArgs,
    OpenRouterModelArgs,
    SelfHostedModelArgs,
)

default_oss_llms_args = {
    "n_retry_server": 4,
    "temperature": 0.01,
}

CLOSED_SOURCE_APIS = [
    "openai",
    "reka",
    "test",
]


import base64
import collections
import importlib
import io
import json
import logging
import os
import re
import time
from copy import deepcopy
from functools import cache
from typing import TYPE_CHECKING, Any, Union
from warnings import warn

import anthropic
import numpy as np
import openai
import tiktoken
import yaml
from PIL import Image


from dataclasses import dataclass
from typing import List

"""
To use this class, you should have the ``openai`` python package installed, and the
environment variable ``OPENAI_API_KEY`` set with your API key.
"""

@dataclass
class PromptTemplate:
    """
    Base class for prompt templates.

    Defines a standard interface for prompt templates, ensuring that they contain
    the required fields for the CustomLLMChatbot.
    """

    system: str
    human: str
    ai: str


import importlib
import logging
import os
import re
import threading
from collections import defaultdict
from contextlib import contextmanager
from dataclasses import dataclass, field
from functools import cache, partial
from typing import Optional

import requests

langchain_community = importlib.util.find_spec("langchain_community")
if langchain_community is not None:
    from langchain_community.callbacks import bedrock_anthropic_callback, openai_info
else:
    bedrock_anthropic_callback = None
    openai_info = None
from litellm import completion_cost, get_model_info


import logging
from importlib import import_module
from pathlib import Path

import bgym

from agentlab.experiments.exp_utils import run_exp
from agentlab.experiments.loop import ExpArgs, yield_all_exp_results

def run_experiments(
    n_jobs,
    exp_args_list: list[ExpArgs],
    study_dir,
    parallel_backend="ray",
    avg_step_timeout=60,
):
    """Run a list of ExpArgs in parallel.

    To ensure optimal parallelism, make sure ExpArgs.depend_on is set correctly
    and the backend is set to dask.


"""Temporary script to get the ray dashboard url for the current experiment.

TODO figure out a more convenient way.
"""

import ray

context = ray.init(address="auto", ignore_reinit_error=True)

print(context)


import logging
import time

import bgym
import ray
from ray.util import state

from agentlab.experiments.exp_utils import _episode_timeout, run_exp

logger = logging.getLogger(__name__)

run_exp = ray.remote(run_exp)

def execute_task_graph(exp_args_list: list[bgym.ExpArgs], avg_step_timeout=60):
    """Execute a task graph in parallel while respecting dependencies using Ray."""

    exp_args_map = {exp_args.exp_id: exp_args for exp_args in exp_args_list}
    task_map = {}

    def get_task(exp_arg: bgym.ExpArgs):


import gzip
import logging
import os
import pickle
import random
import uuid
from abc import ABC, abstractmethod
from concurrent.futures import ProcessPoolExecutor
from dataclasses import asdict, dataclass
from datetime import datetime
from multiprocessing import Manager, Pool, Queue
from pathlib import Path

import bgym
from bgym import DEFAULT_BENCHMARKS, Benchmark
from slugify import slugify

from agentlab.agents.agent_args import AgentArgs
from agentlab.analyze import inspect_results
from agentlab.benchmarks.abstract_env import AbstractEnvArgs
from agentlab.experiments import reproducibility_util as repro


"""Dirty script to visualize the dependency graph of a benchmark, e.g. webarena, vsisualwebarena,
etc. You may have to detust it to make it work for you."""

import math

import bgym
import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
from bgym import DEFAULT_BENCHMARKS

def clean_dict(dependency_dict: dict[str, list[str]]) -> dict[str, list[str]]:
    new_dep = {}
    for key, deps in dependency_dict.items():
        new_key = key.split(".")[-1]

        new_dep[new_key] = [dep.split(".")[-1] for dep in deps]
    return new_dep


import csv
import logging
import os
import platform
from datetime import datetime
from importlib import metadata
from pathlib import Path

import bgym
import pandas as pd
from bgym import Benchmark
from git import InvalidGitRepositoryError, Repo
from git.config import GitConfigParser

import agentlab
from agentlab.experiments.exp_utils import RESULTS_DIR

def _get_repo(module):
    return Repo(Path(module.__file__).resolve().parent, search_parent_directories=True)


import gzip
import importlib.metadata
import json
import logging
import os
import pickle
import re
import sys
import time
import traceback
import uuid
from abc import ABC, abstractmethod
from collections import defaultdict
from dataclasses import asdict, dataclass, field, is_dataclass
from datetime import datetime
from pathlib import Path
from typing import Optional

import gymnasium as gym
import numpy as np
from browsergym.core.chat import Chat


import copy
from abc import ABC
from dataclasses import fields, is_dataclass
from itertools import product
from typing import Any

import numpy as np

class CrossProd:
    """Use to specify that this will be part of a cross product"""

    def __init__(self, elements):
        self.elements = elements

class Distribution(ABC):
    """Generic Class to identify that this is a distribution"""

    def sample(self):
        pass


import pandas as pd
from openai import OpenAI

if __name__ == "__main__":
    models = OpenAI().models.list()
    df = pd.DataFrame([dict(model) for model in models.data])

    # Filter GPT models or o1 models
    # df = df[df["id"].str.contains("gpt") | df["id"].str.contains("o1")]

    # Convert Unix timestamps to dates (YYYY-MM-DD) and remove time
    df["created"] = pd.to_datetime(df["created"], unit="s").dt.date
    df.sort_values(by="created", inplace=True)
    # Print all entries

    # print all entries
    print(df.to_string(index=False))


import logging
import os
import signal
import sys
from contextlib import contextmanager
from pathlib import Path
from time import sleep, time

from tqdm import tqdm

from agentlab.experiments.loop import ExpArgs, yield_all_exp_results

logger = logging.getLogger(__name__)  # Get logger based on module name

# TODO move this to a more appropriate place
RESULTS_DIR = os.environ.get("AGENTLAB_EXP_ROOT", None)
if RESULTS_DIR is None:
    RESULTS_DIR = os.environ.get("UI_COPILOT_RESULTS_DIR", None)
if RESULTS_DIR is None:
    logging.info("$AGENTLAB_EXP_ROOT is not defined, Using $HOME/agentlab_results.")


from copy import deepcopy
from dataclasses import dataclass
import os
import sys
from browsergym.webarena.instance import WebArenaInstance

class BaseServer:
    """Base class for server instances.

    Behaves like an identity function for running in parallel on servers that don't need multiple
    instances.
    """

    def init(self):
        pass

@dataclass
class WebArenaInstanceVars(BaseServer):
    base_url: str


"""
This script will leverage an old study to reproduce it on the same tasks and
same seeds. Instead of calling the LLM it will reuse the responses from the old
llm. Load the study in agent-xray and look at the Agent Info HTML to compare
the diff in HTML format.
"""

from agentlab.agents.generic_agent.reproducibility_agent import reproduce_study
from agentlab.experiments.exp_utils import RESULTS_DIR

if __name__ == "__main__":

    # replace by your study name
    old_study = "2024-06-03_12-28-51_final_run_miniwob_llama3-70b"

    study = reproduce_study(RESULTS_DIR / old_study)
    n_jobs = 1

    study.run(n_jobs=n_jobs, parallel_backend="joblib", strict_reproducibility=False)


import io
import xml.etree.ElementTree as ET
from typing import Tuple, List

from PIL import Image, ImageDraw, ImageFont

def find_leaf_nodes(xlm_file_str):
    if not xlm_file_str:
        return []

    root = ET.fromstring(xlm_file_str)

    # Recursive function to traverse the XML tree and collect leaf nodes
    def collect_leaf_nodes(node, leaf_nodes):
        # If the node has no children, it is a leaf node, add it to the list
        if not list(node):
            leaf_nodes.append(node)
        # If the node has children, recurse on each child
        for child in node:
            collect_leaf_nodes(child, leaf_nodes)


import logging
import time

from tapeagents.core import Action, Observation, StopStep
from tapeagents.environment import ToolCollectionEnvironment
from tapeagents.tools.base import StatefulTool, Tool

from agentlab.benchmarks.abstract_env import AbstractEnv

logger = logging.getLogger(__name__)

class MultiToolGym(AbstractEnv):
    def __init__(self, tools: list[Tool | StatefulTool], max_turns: int = 50):
        self._env = ToolCollectionEnvironment(tools)
        self._actions = self._env.actions()
        self.max_turns = max_turns
        self._turns = 0

    def reset(self):
        self._env.reset()


import ast
import importlib.util
import json
import logging
import os
import time
from copy import deepcopy
from dataclasses import dataclass
from io import BytesIO
from pathlib import Path
from typing import Any, Literal

import numpy as np
from bgym import AbstractActionSet
from dataclasses_json import DataClassJsonMixin
from PIL import Image

from agentlab.benchmarks.abstract_env import (
    AbstractBenchmark,
    AbstractEnv,
    AbstractEnvArgs,


"""Tiny benchmark setup helpers.

Currently supports MiniWob++: clones the repo at a pinned commit and writes
MINIWOB_URL to .env. Designed to be minimal and easy to maintain.
"""

from __future__ import annotations

import logging
import os
import pathlib
from typing import Optional

logger = logging.getLogger(__name__)

def _ensure_repo(repo_url: str, clone_dir: pathlib.Path, commit: Optional[str] = None) -> None:
    """Clone repo if missing and optionally checkout a commit (minimal, shell-only).

    Args:
        repo_url: URL of the git repository to clone.


import logging
import os
import re
import shutil
import string
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal, Self

import datasets
import hydra
import podman
import tapeagents.config
from omegaconf import DictConfig
from pdf2image import convert_from_path
from pydantic import ConfigDict, Field
from tapeagents.core import Action, Observation, StopStep, Thought
from tapeagents.environment import ContainerExecutor, StatefulTool, Tool
from tapeagents.steps import ImageObservation
from tapeagents.tools.simple_browser import SimpleTextBrowser


import time
from abc import ABC, abstractmethod
from functools import wraps

import gymnasium as gym
from dataclasses_json import DataClassJsonMixin
from pydantic import BaseModel

class AbstractEnvArgs(DataClassJsonMixin):
    @abstractmethod
    def make_env(self, action_mapping, exp_dir, exp_task_kwargs) -> "AbstractEnv":
        """Create an instance of the environment with the arguments stored in this object.

        Args:
            action_mapping (dict[str,str]): mapping from the agent's action space to the environment's action space
                see AbstractActionSet.to_python_code from BrowserGym for an example
            exp_dir (str): directory where the experiment is stored
            exp_task_kwargs (dict[str,Any]): additional arguments for the environment

        Returns:


from copy import deepcopy
from dataclasses import asdict, dataclass
from functools import partial

import bgym
from browsergym.experiments.agent import Agent, AgentInfo
from browsergym.utils.obs import flatten_axtree_to_str, flatten_dom_to_str, overlay_som, prune_html

from agentlab.agents.agent_args import AgentArgs
from agentlab.llm.chat_api import BaseModelArgs
from agentlab.llm.llm_utils import ParseError, image_to_png_base64_url, parse_html_tags_raise, retry
from agentlab.llm.tracking import cost_tracker_decorator

@dataclass
class DebugAgentArgs(AgentArgs):

    def __post_init__(self):
        try:  # some attributes might be temporarily args.CrossProd for hyperparameter generation
            self.agent_name = f"debug".replace("/", "_")
        except AttributeError:


import copy

from PIL import Image, ImageDraw
from playwright.sync_api import Page

from agentlab.analyze import overlay_utils
from agentlab.llm.llm_utils import img_to_base_64

def draw_mouse_pointer(image: Image.Image, x: int, y: int) -> Image.Image:
    """
    Draws a semi-transparent mouse pointer at (x, y) on the image.
    Returns a new image with the pointer drawn.

    Args:
        image: The image to draw the mouse pointer on.
        x: The x coordinate for the mouse pointer.
        y: The y coordinate for the mouse pointer.

    Returns:
        A new image with the mouse pointer drawn.


import bgym
from bgym import AbstractAgentArgs, Benchmark

class AgentArgs(AbstractAgentArgs):
    """Base class for agent arguments for instantiating an agent.

    Define agent arguments as dataclass variables of this class. For example:

    class MyAgentArgs(AgentArgs):
        my_arg: str = "default_value"
        my_other_arg: int = 42

    Note: for working properly with AgentXRay, the arguments need to be serializable and hasable.
    """

    def set_benchmark(self, benchmark: Benchmark, demo_mode: bool):
        """Optional method to set benchmark specific flags.

        This allows the agent to have minor adjustments based on the benchmark.
        E.g. using a benchmark specific action space. Or letting the agent see


import abc
import logging
import platform
import time
from copy import copy, deepcopy
from dataclasses import asdict, dataclass
from textwrap import dedent
from typing import Literal
from warnings import warn

import bgym
from bgym import HighLevelActionSetArgs
from browsergym.core.action.base import AbstractActionSet
from browsergym.utils.obs import flatten_axtree_to_str, flatten_dom_to_str, overlay_som, prune_html

from agentlab.llm.llm_utils import (
    BaseMessage,
    ParseError,
    count_tokens,
    extract_code_blocks,
    image_to_jpg_base64_url,


"""
AgentLab's pre-implemented agents.

This module contains the agent implementations for AgentLab. With currently:

- GenericAgent: Our baseline agent for evaluation

- MostBasicAgent: A basic agent for learning our framework

- TapeAgent: An agent that uses the Tape data structure to perform actions

- VisualWebArenaAgent: An implementation of the agent used in WebArena and VisualWebArena
"""


"""
Basic flags and agent configurations for generic agents.
"""

import bgym
from bgym import HighLevelActionSetArgs

from agentlab.agents import dynamic_prompting as dp
from agentlab.experiments import args
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT

from .generic_agent import GenericAgentArgs
from .generic_agent_prompt import GenericPromptFlags
from .tmlr_config import BASE_FLAGS

FLAGS_CUSTOM = GenericPromptFlags(
    obs=dp.ObsFlags(
        use_html=False,
        use_ax_tree=True,
        use_focused_element=True,
        use_error_logs=True,


"""
Specific configurations for our 2024 TMLR submission.
"""

from copy import deepcopy

from agentlab.agents import dynamic_prompting as dp
from agentlab.experiments import args
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT

from .generic_agent import GenericAgentArgs
from .generic_agent_prompt import GenericPromptFlags

BASE_FLAGS = GenericPromptFlags(
    obs=dp.ObsFlags(
        use_html=False,
        use_ax_tree=True,
        use_focused_element=True,
        use_error_logs=True,
        use_history=True,
        use_past_error_logs=False,


"""
Prompt builder for GenericAgent

It is based on the dynamic_prompting module from the agentlab package.
"""

import logging
from dataclasses import dataclass

from browsergym.core import action
from browsergym.core.action.base import AbstractActionSet

from agentlab.agents import dynamic_prompting as dp
from agentlab.llm.llm_utils import HumanMessage, parse_html_tags_raise

@dataclass
class GenericPromptFlags(dp.Flags):
    """
    A class to represent various flags used to control features in an application.


"""
Baseline agent for all ServiceNow papers

This module contains the GenericAgent class, which is the baseline agent for all ServiceNow papers. \
It is a simple agent that can be ran OOB on all BrowserGym environments. It is also shipped with \
a few configurations that can be used to run it on different environments.
"""

from .agent_configs import (
    AGENT_3_5,
    AGENT_8B,
    AGENT_37_SONNET,
    AGENT_CLAUDE_SONNET_35,
    AGENT_CLAUDE_SONNET_35_VISION,
    AGENT_CUSTOM,
    AGENT_LLAMA3_70B,
    AGENT_LLAMA4_17B_INSTRUCT,
    AGENT_LLAMA31_70B,
    CHAT_MODEL_ARGS_DICT,
    RANDOM_SEARCH_AGENT,
    AGENT_4o,


"""
An agent that reproduces exactly the same traces as GenericAgent, to compare the results.

This module contains the classes and functions to reproduce the results of a
study. It is used to create a new study that will run the same experiments as
the original study, but with a reproducibility agent that will mimic the same
answers as the original agent.

Stats are collected to compare the original agent's answers with the new agent's
answers. Load the this reproducibility study in agent-xray to compare the results.
"""

import difflib
import logging
import time
from copy import copy
from dataclasses import dataclass
from pathlib import Path

import bgym


"""
GenericAgent implementation for AgentLab

This module defines a `GenericAgent` class and its associated arguments for use in the AgentLab framework. \
The `GenericAgent` class is designed to interact with a chat-based model to determine actions based on \
observations. It includes methods for preprocessing observations, generating actions, and managing internal \
state such as plans, memories, and thoughts. The `GenericAgentArgs` class provides configuration options for \
the agent, including model arguments and flags for various behaviors.
"""

from copy import deepcopy
from dataclasses import asdict, dataclass
from functools import partial
from warnings import warn

import bgym
from bgym import Benchmark
from browsergym.experiments.agent import Agent, AgentInfo

from agentlab.agents import dynamic_prompting as dp
from agentlab.agents.agent_args import AgentArgs


from typing_extensions import Protocol

from agentlab.agents.agent_args import AgentArgs

class MultiCandidateAgent(Protocol):
    """
    Protocol for agents that generate multiple candidates for get_action.

    This protocol defines the contract for agents that can generate
    multiple candidate actions and allow selection of one of them for execution.
    """

    def get_candidate_generations(
        self, obs: dict, hint: list[str] | None = None, n_candidates: int = 3
    ) -> "list[dict]":
        """
        Generate multiple candidate actions for the given observation.

        You can pass extra info in agent_info to update internal state of the
        agent based on the selected candidate. Your internal state management


"""
Console launcher for the Human-in-the-Loop Generic Agent UI.

Usage (installed entry point):
    agentlab-mentor --benchmark miniwob --task-name miniwob.book-flight --seed 123 --no-headless

This will run a Study with the MultipleProposalGenericAgent and the selected task.
"""

from __future__ import annotations

import argparse
import logging
from pathlib import Path

import bgym

from agentlab.agents.hitl_agent.generic_human_guided_agent import get_base_agent
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.study import Study


import re
from dataclasses import asdict, dataclass
from typing import Dict, List

from browsergym.experiments.agent import AgentInfo

from agentlab.agents import dynamic_prompting as dp
from agentlab.agents.generic_agent.generic_agent import GenericAgent, GenericAgentArgs
from agentlab.agents.generic_agent.generic_agent_prompt import MainPrompt
from agentlab.llm.llm_utils import Discussion, HumanMessage, SystemMessage

class CandidatesGeneration(dp.PromptElement):
    # Ask for multiple alternatives; each candidate must contain <think> and <action>.
    def __init__(self, hint: list[str] | None = None, n_candidates=3) -> None:
        self.hint = hint
        self.n_candidates = n_candidates
        self.hint_prompt = "\n".join(f"{i}. {c}" for i, c in enumerate(hint, 1)) if hint else ""
        super().__init__(True)
        self._prompt = [
            dict(


import json
import logging
from importlib import resources
from queue import Queue
from typing import Dict, List, Optional

import playwright.sync_api
from browsergym.core import _get_global_playwright
from pydantic import BaseModel, Field

logger = logging.getLogger(__name__)

HINT_LABELING_DIR = resources.files("agentlab.agents.hitl_agent.hint_labelling_ui_files")

class HintLabelingInputs(BaseModel):
    goal: str
    error_feedback: str = ""
    screenshot: str  # base64 screenshot (original/current)
    screenshots: List[str] = Field(default_factory=list)  # list of base64 screenshots for hover
    axtree: str


from dataclasses import dataclass
from typing import Optional

import bgym
import playwright
from browsergym.experiments.agent import Agent

from agentlab.agents.agent_args import AgentArgs
from agentlab.agents.agent_utils import overlay_action
from agentlab.agents.hitl_agent.base_multi_candidate_agent import MultiCandidateAgent
from agentlab.agents.hitl_agent.hint_labelling import (
    HintLabeling,
    HintLabelingInputs,
)
from agentlab.llm.llm_utils import img_to_base_64
from agentlab.llm.tracking import cost_tracker_decorator

class HumanInTheLoopAgent(Agent):

    def __init__(


import base64
import copy
import io
import re
from dataclasses import Field, asdict, dataclass
from typing import Dict, List

import bgym
import numpy as np
from browsergym.experiments.agent import AgentInfo
from PIL import Image

from agentlab.agents import dynamic_prompting as dp
from agentlab.agents.agent_utils import overlay_action
from agentlab.agents.generic_agent.generic_agent import GenericAgent, GenericAgentArgs
from agentlab.agents.generic_agent.generic_agent_prompt import MainPrompt
from agentlab.agents.hitl_agent.hint_labelling import (
    HintLabeling,
    HintLabelingInputs,
)
from agentlab.llm.llm_utils import (


import sys

from agentlab.agents.tool_use_agent.tool_use_agent import *

# for backward compatibility of unpickling
sys.modules[__name__ + ".multi_tool_agent"] = sys.modules[__name__]


import fnmatch
import json
import logging
import os
import random
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from copy import copy
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any, Literal

import bgym
import numpy as np
import pandas as pd
import requests
from bgym import Benchmark as BgymBenchmark
from browsergym.core.observation import extract_screenshot
from browsergym.utils.obs import (
    flatten_axtree_to_str,


import base64
import importlib.resources
import io
from copy import deepcopy
from dataclasses import dataclass
from functools import partial
from typing import Any, Literal

import numpy as np
import PIL.Image
from browsergym.core.action.highlevel import HighLevelActionSet
from browsergym.experiments import Agent, AgentInfo
from browsergym.experiments.benchmark import Benchmark, HighLevelActionSetArgs
from browsergym.utils.obs import overlay_som

from agentlab.llm.base_api import AbstractChatModel
from agentlab.llm.chat_api import BaseModelArgs, make_system_message, make_user_message
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT
from agentlab.llm.llm_utils import ParseError, extract_code_blocks, retry
from agentlab.llm.tracking import cost_tracker_decorator


# Best-attempt reproduction of original prompts the VisualWebArena agent

TEMPLATES = {}

# https://github.com/web-arena-x/visualwebarena/blob/89f5af29305c3d1e9f97ce4421462060a70c9a03/agent/prompts/raw/p_cot_id_actree_3s.py#L1
TEMPLATES["axtree"] = {
    "intro": """\
You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.

Here's the information you'll have:
The user's objective: This is the task you're trying to complete.
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
The current web page's URL: This is the page you're currently navigating.
The open tabs: These are the tabs you have open.
The previous action: This is the action you just performed. It may be helpful to track your progress.

{action_space_description}

To be successful, it is very important to follow the following rules:
1. You should only issue an action that is valid given the current observation
2. You should only issue one action at a time.


import logging
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, Any

import bgym

from agentlab.agents.agent_args import AgentArgs
from agentlab.experiments.loop import ExpArgs
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT
from agentlab.llm.llm_utils import (
    Discussion,
    HumanMessage,
    ParseError,
    SystemMessage,
    extract_code_blocks,
    retry,
)
from agentlab.llm.tracking import cost_tracker_decorator

if TYPE_CHECKING:
    from agentlab.llm.chat_api import BaseModelArgs


import bgym
from bgym import HighLevelActionSetArgs

import agentlab.agents.dynamic_prompting as dp
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT

from .visual_agent import VisualAgentArgs
from .visual_agent_prompts import PromptFlags

# the other flags are ignored for this agent.
DEFAULT_OBS_FLAGS = dp.ObsFlags(
    use_tabs=True,  # will be overridden by the benchmark when set_benchmark is called after initalizing the agent
    use_error_logs=True,
    use_past_error_logs=False,
    use_screenshot=True,
    use_som=False,
    openai_vision_detail="auto",
)

DEFAULT_ACTION_FLAGS = dp.ActionFlags(
    action_set=HighLevelActionSetArgs(subsets=["coord"]),


"""
GenericAgent implementation for AgentLab

This module defines a `GenericAgent` class and its associated arguments for use in the AgentLab framework. \
The `GenericAgent` class is designed to interact with a chat-based model to determine actions based on \
observations. It includes methods for preprocessing observations, generating actions, and managing internal \
state such as plans, memories, and thoughts. The `GenericAgentArgs` class provides configuration options for \
the agent, including model arguments and flags for various behaviors.
"""

from dataclasses import asdict, dataclass

import bgym
from bgym import Benchmark
from browsergym.experiments.agent import Agent, AgentInfo

from agentlab.agents import dynamic_prompting as dp
from agentlab.agents.agent_args import AgentArgs
from agentlab.llm.chat_api import BaseModelArgs
from agentlab.llm.llm_utils import Discussion, ParseError, SystemMessage, retry
from agentlab.llm.tracking import cost_tracker_decorator


"""
Prompt builder for GenericAgent

It is based on the dynamic_prompting module from the agentlab package.
"""

import logging
from dataclasses import dataclass
import bgym

from browsergym.core.action.base import AbstractActionSet

from agentlab.agents import dynamic_prompting as dp
from agentlab.llm.llm_utils import BaseMessage, HumanMessage, image_to_jpg_base64_url

@dataclass
class PromptFlags(dp.Flags):
    """
    A class to represent various flags used to control features in an application.
    """


import logging
from dataclasses import dataclass
from typing import Literal

import bgym
import hydra
from omegaconf import DictConfig
from pydantic import Field
from tapeagents.agent import Agent
from tapeagents.core import Action, Observation, StopStep, TapeMetadata, Thought
from tapeagents.core import Tape as BaseTape

from agentlab.agents.agent_args import AgentArgs

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

class ExtendedMetadata(TapeMetadata):
    name: str = ""
    task: dict = {}


import json
from dataclasses import asdict, is_dataclass

import numpy as np
from tapeagents.core import Step, StepMetadata
from tapeagents.dialog_tape import AssistantStep, AssistantThought
from tapeagents.io import save_json_tape, save_tape_images

from agentlab.agents.tapeagent.agent import DictObservation, Tape, TapeAgent

__all__ = ["as_tape", "save_tape", "TapeAgent", "Tape"]

def as_tape(steps_info: list) -> Tape:
    """
    Create a Tape object from the steps info.

    Args:
        steps_info: list of StepInfo objects.

    Returns:


import logging
import os

from agentlab.agents.tapeagent.agent import TapeAgentArgs, load_config
from agentlab.benchmarks.gaia import GaiaBenchmark, stop_old_sandbox
from agentlab.experiments.study import make_study

fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

if __name__ == "__main__":
    config = load_config("gaia_l1")
    study = make_study(
        benchmark=GaiaBenchmark.from_config(config),  # type: ignore
        agent_args=TapeAgentArgs(agent_name=config.name, config=config),
        comment=config.comment,
        logging_level=logging.INFO,
        logging_level_stdout=logging.INFO,
    )
    stop_old_sandbox()
    if os.environ.get("AGENTLAB_DEBUG"):