import bgym
import pytest
import ray
from flaky import flaky

from agentlab.experiments.exp_utils import MockedExpArgs, add_dependencies
from agentlab.experiments.graph_execution_ray import execute_task_graph

TASK_TIME = 3

@flaky(max_runs=3, min_passes=1)
def test_execute_task_graph():
    # Define a list of ExpArgs with dependencies
    exp_args_list = [
        MockedExpArgs(exp_id="task1", depends_on=[]),
        MockedExpArgs(exp_id="task2", depends_on=["task1"]),
        MockedExpArgs(exp_id="task3", depends_on=["task1"]),
        MockedExpArgs(exp_id="task4", depends_on=["task2", "task3"]),
    ]

    ray.init(num_cpus=4)
    results = execute_task_graph(exp_args_list)
    ray.shutdown()

    exp_args_list = [results[task_id] for task_id in ["task1", "task2", "task3", "task4"]]

    # Verify that all tasks were executed in the proper order
    assert exp_args_list[0].start_time < exp_args_list[1].start_time
    assert exp_args_list[0].start_time < exp_args_list[2].start_time
    assert exp_args_list[1].end_time < exp_args_list[3].start_time
    assert exp_args_list[2].end_time < exp_args_list[3].start_time

    # Verify that parallel tasks (task2 and task3) started within a short time of each other
    # TODO: replace with non flaky check
    # parallel_start_diff = abs(exp_args_list[1].start_time - exp_args_list[2].start_time)
    # print(f"parallel_start_diff: {parallel_start_diff}")
    # assert parallel_start_diff < 2, "Parallel tasks should start within 2 seconds of each other"

    # Ensure that the entire task graph took the expected amount of time
    # TODO: replace with non flaky check
    # total_time = exp_args_list[-1].end_time - exp_args_list[0].start_time
    # # Since the critical path involves at least 1.5 seconds of work
    # assert total_time >= TASK_TIME * 3, "Total time should be at least 3 times the task time"

def test_add_dependencies():
    # Prepare a simple list of ExpArgs

    def make_exp_args(task_name, exp_id):
        return bgym.ExpArgs(
            agent_args=None, env_args=bgym.EnvArgs(task_name=task_name), exp_id=exp_id
        )

    exp_args_list = [
        make_exp_args("task1", "1"),
        make_exp_args("task2", "2"),
        make_exp_args("task3", "3"),
    ]

    # Define simple task_dependencies
    task_dependencies = {"task1": ["task2"], "task2": [], "task3": ["task1"]}

    # Call the function
    modified_list = add_dependencies(exp_args_list, task_dependencies)

    # Verify dependencies
    assert modified_list[0].depends_on == ("2",)  # task1 depends on task2
    assert modified_list[1].depends_on == ()  # task2 has no dependencies
    assert modified_list[2].depends_on == ("1",)  # task3 depends on task1

    # assert raise if task_dependencies is wrong
    task_dependencies = {"task1": ["task2"], "task2": [], "task4": ["task3"]}
    with pytest.raises(ValueError):
        add_dependencies(exp_args_list, task_dependencies)

if __name__ == "__main__":
    test_execute_task_graph()
    # test_add_dependencies()


import bgym
import pytest
import ray
from flaky import flaky

from agentlab.experiments.exp_utils import MockedExpArgs, add_dependencies
from agentlab.experiments.graph_execution_ray import execute_task_graph

TASK_TIME = 3

@flaky(max_runs=3, min_passes=1)
def test_execute_task_graph():
    # Define a list of ExpArgs with dependencies
    exp_args_list = [
        MockedExpArgs(exp_id="task1", depends_on=[]),
        MockedExpArgs(exp_id="task2", depends_on=["task1"]),
        MockedExpArgs(exp_id="task3", depends_on=["task1"]),
        MockedExpArgs(exp_id="task4", depends_on=["task2", "task3"]),
    ]

    ray.init(num_cpus=4)
    results = execute_task_graph(exp_args_list)
    ray.shutdown()

    exp_args_list = [results[task_id] for task_id in ["task1", "task2", "task3", "task4"]]

    # Verify that all tasks were executed in the proper order
    assert exp_args_list[0].start_time < exp_args_list[1].start_time
    assert exp_args_list[0].start_time < exp_args_list[2].start_time
    assert exp_args_list[1].end_time < exp_args_list[3].start_time
    assert exp_args_list[2].end_time < exp_args_list[3].start_time

    # Verify that parallel tasks (task2 and task3) started within a short time of each other
    # TODO: replace with non flaky check
    # parallel_start_diff = abs(exp_args_list[1].start_time - exp_args_list[2].start_time)
    # print(f"parallel_start_diff: {parallel_start_diff}")
    # assert parallel_start_diff < 2, "Parallel tasks should start within 2 seconds of each other"

    # Ensure that the entire task graph took the expected amount of time
    # TODO: replace with non flaky check
    # total_time = exp_args_list[-1].end_time - exp_args_list[0].start_time
    # # Since the critical path involves at least 1.5 seconds of work
    # assert total_time >= TASK_TIME * 3, "Total time should be at least 3 times the task time"

def test_add_dependencies():
    # Prepare a simple list of ExpArgs

    def make_exp_args(task_name, exp_id):
        return bgym.ExpArgs(
            agent_args=None, env_args=bgym.EnvArgs(task_name=task_name), exp_id=exp_id


    # Verify that all tasks were executed in the proper order
    assert exp_args_list[0].start_time < exp_args_list[1].start_time
    assert exp_args_list[0].start_time < exp_args_list[2].start_time
    assert exp_args_list[1].end_time < exp_args_list[3].start_time
    assert exp_args_list[2].end_time < exp_args_list[3].start_time

    # Verify that parallel tasks (task2 and task3) started within a short time of each other
    # TODO: replace with non flaky check
    # parallel_start_diff = abs(exp_args_list[1].start_time - exp_args_list[2].start_time)
    # print(f"parallel_start_diff: {parallel_start_diff}")
    # assert parallel_start_diff < 2, "Parallel tasks should start within 2 seconds of each other"

    # Ensure that the entire task graph took the expected amount of time
    # TODO: replace with non flaky check
    # total_time = exp_args_list[-1].end_time - exp_args_list[0].start_time
    # # Since the critical path involves at least 1.5 seconds of work
    # assert total_time >= TASK_TIME * 3, "Total time should be at least 3 times the task time"

def test_add_dependencies():
    # Prepare a simple list of ExpArgs

    def make_exp_args(task_name, exp_id):
        return bgym.ExpArgs(
            agent_args=None, env_args=bgym.EnvArgs(task_name=task_name), exp_id=exp_id
        )

    exp_args_list = [
        make_exp_args("task1", "1"),
        make_exp_args("task2", "2"),
        make_exp_args("task3", "3"),
    ]

    # Define simple task_dependencies
    task_dependencies = {"task1": ["task2"], "task2": [], "task3": ["task1"]}

    # Call the function
    modified_list = add_dependencies(exp_args_list, task_dependencies)

    # Verify dependencies
    assert modified_list[0].depends_on == ("2",)  # task1 depends on task2
    assert modified_list[1].depends_on == ()  # task2 has no dependencies
    assert modified_list[2].depends_on == ("1",)  # task3 depends on task1

    # assert raise if task_dependencies is wrong
    task_dependencies = {"task1": ["task2"], "task2": [], "task4": ["task3"]}
    with pytest.raises(ValueError):
        add_dependencies(exp_args_list, task_dependencies)

if __name__ == "__main__":
    test_execute_task_graph()
    # test_add_dependencies()


    assert exp_args_list[0].start_time < exp_args_list[2].start_time
    assert exp_args_list[1].end_time < exp_args_list[3].start_time
    assert exp_args_list[2].end_time < exp_args_list[3].start_time

    # Verify that parallel tasks (task2 and task3) started within a short time of each other
    # TODO: replace with non flaky check
    # parallel_start_diff = abs(exp_args_list[1].start_time - exp_args_list[2].start_time)
    # print(f"parallel_start_diff: {parallel_start_diff}")
    # assert parallel_start_diff < 2, "Parallel tasks should start within 2 seconds of each other"

    # Ensure that the entire task graph took the expected amount of time
    # TODO: replace with non flaky check
    # total_time = exp_args_list[-1].end_time - exp_args_list[0].start_time
    # # Since the critical path involves at least 1.5 seconds of work
    # assert total_time >= TASK_TIME * 3, "Total time should be at least 3 times the task time"

def test_add_dependencies():
    # Prepare a simple list of ExpArgs

    def make_exp_args(task_name, exp_id):
        return bgym.ExpArgs(
            agent_args=None, env_args=bgym.EnvArgs(task_name=task_name), exp_id=exp_id
        )

    exp_args_list = [
        make_exp_args("task1", "1"),
        make_exp_args("task2", "2"),
        make_exp_args("task3", "3"),
    ]

    # Define simple task_dependencies
    task_dependencies = {"task1": ["task2"], "task2": [], "task3": ["task1"]}

    # Call the function
    modified_list = add_dependencies(exp_args_list, task_dependencies)

    # Verify dependencies
    assert modified_list[0].depends_on == ("2",)  # task1 depends on task2
    assert modified_list[1].depends_on == ()  # task2 has no dependencies
    assert modified_list[2].depends_on == ("1",)  # task3 depends on task1

    # assert raise if task_dependencies is wrong
    task_dependencies = {"task1": ["task2"], "task2": [], "task4": ["task3"]}


from ast import mod
from dataclasses import dataclass
from agentlab.experiments.args import (
    expand_cross_product,
    CrossProd,
    Choice,
    make_progression_study,
    sample_args,
    make_ablation_study,
)

@dataclass
class LLMArgsTest:
    model_name: str = "model1"
    temperature: float = 0.1

@dataclass
class ExpArgsTest:
    llm_args: LLMArgsTest
    task_name: str = "task1"
    n_episode: int = 10

def test_cross_product():
    exp_args = ExpArgsTest(
        n_episode=CrossProd([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args)
    assert len(expanded_args_list) == 6

    variables = [(args.n_episode, args.llm_args.model_name) for args in expanded_args_list]
    variables.sort()
    assert variables == [
        (1, "model1"),
        (1, "model2"),
        (2, "model1"),
        (2, "model2"),
        (3, "model1"),
        (3, "model2"),
    ]

def test_cross_product_dict():
    exp_args_dict = dict(
        n_episode=CrossProd([1, 2, 3]),
        llm_args=dict(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args_dict)
    assert len(expanded_args_list) == 6

    variables = [(args["n_episode"], args["llm_args"]["model_name"]) for args in expanded_args_list]
    variables.sort()
    assert variables == [
        (1, "model1"),
        (1, "model2"),
        (2, "model1"),
        (2, "model2"),
        (3, "model1"),
        (3, "model2"),
    ]

def test_sample():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=Choice(["model1", "model2"]),
        ),
    )
    exp_args_sample = sample_args(exp_args, 3)
    assert len(exp_args_sample) == 3
    for exp_args in exp_args_sample:
        assert exp_args.n_episode in [1, 2, 3]
        assert exp_args.llm_args.model_name in ["model1", "model2"]
        assert exp_args.llm_args.temperature == 0.1

def test_sample_and_cross_prod():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    def assert_ok(exp_args_list):
        assert len(exp_args_list) == 6
        for exp_args in exp_args_list:
            assert exp_args.n_episode in [1, 2, 3]
            assert exp_args.llm_args.model_name in ["model1", "model2"]
            assert exp_args.llm_args.temperature == 0.1

    exp_args_list_1 = expand_cross_product(sample_args(exp_args, 3))
    assert_ok(exp_args_list_1)
    exp_args_list_2 = sample_args(expand_cross_product(exp_args), 3)
    assert_ok(exp_args_list_2)

def test_make_progression_study():
    ablation = make_progression_study(
        start_point=LLMArgsTest(
            model_name="model1",
            temperature=0.1,
        ),
        changes=[
            ("model_name", "model2"),
            ("temperature", 0.2),
        ],
    )

    configs = expand_cross_product(
        ExpArgsTest(
            n_episode=CrossProd([1, 2]),
            llm_args=ablation,
        )
    )

    assert len(configs) == 6
    params = [(config.llm_args.model_name, config.llm_args.temperature) for config in configs]
    params = list(set(params))
    params.sort()

    assert params == [("model1", 0.1), ("model2", 0.1), ("model2", 0.2)]

def test_make_ablation_study():
    ablation = make_ablation_study(
        start_point=LLMArgsTest(
            model_name="model1",
            temperature=0.1,
        ),
        changes=[
            ("model_name", "model2"),
            ("temperature", 0.2),
        ],
    )

    configs = expand_cross_product(
        ExpArgsTest(
            n_episode=CrossProd([1, 2]),
            llm_args=ablation,
        )
    )

    assert len(configs) == 6
    params = [(config.llm_args.model_name, config.llm_args.temperature) for config in configs]
    params = list(set(params))
    params.sort()

    assert params == [("model1", 0.1), ("model1", 0.2), ("model2", 0.1)]


LLMArgsTest

from ast import mod
from dataclasses import dataclass
from agentlab.experiments.args import (
    expand_cross_product,
    CrossProd,
    Choice,
    make_progression_study,
    sample_args,
    make_ablation_study,
)

@dataclass
class LLMArgsTest:
    model_name: str = "model1"
    temperature: float = 0.1

@dataclass
class ExpArgsTest:
    llm_args: LLMArgsTest
    task_name: str = "task1"
    n_episode: int = 10

def test_cross_product():
    exp_args = ExpArgsTest(
        n_episode=CrossProd([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args)
    assert len(expanded_args_list) == 6


ExpArgsTest

from ast import mod
from dataclasses import dataclass
from agentlab.experiments.args import (
    expand_cross_product,
    CrossProd,
    Choice,
    make_progression_study,
    sample_args,
    make_ablation_study,
)

@dataclass
class LLMArgsTest:
    model_name: str = "model1"
    temperature: float = 0.1

@dataclass
class ExpArgsTest:
    llm_args: LLMArgsTest
    task_name: str = "task1"
    n_episode: int = 10

def test_cross_product():
    exp_args = ExpArgsTest(
        n_episode=CrossProd([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args)
    assert len(expanded_args_list) == 6

    variables = [(args.n_episode, args.llm_args.model_name) for args in expanded_args_list]
    variables.sort()
    assert variables == [
        (1, "model1"),
        (1, "model2"),
        (2, "model1"),
        (2, "model2"),


    Choice,
    make_progression_study,
    sample_args,
    make_ablation_study,
)

@dataclass
class LLMArgsTest:
    model_name: str = "model1"
    temperature: float = 0.1

@dataclass
class ExpArgsTest:
    llm_args: LLMArgsTest
    task_name: str = "task1"
    n_episode: int = 10

def test_cross_product():
    exp_args = ExpArgsTest(
        n_episode=CrossProd([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args)
    assert len(expanded_args_list) == 6

    variables = [(args.n_episode, args.llm_args.model_name) for args in expanded_args_list]
    variables.sort()
    assert variables == [
        (1, "model1"),
        (1, "model2"),
        (2, "model1"),
        (2, "model2"),
        (3, "model1"),
        (3, "model2"),
    ]

def test_cross_product_dict():
    exp_args_dict = dict(
        n_episode=CrossProd([1, 2, 3]),
        llm_args=dict(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args_dict)
    assert len(expanded_args_list) == 6

    variables = [(args["n_episode"], args["llm_args"]["model_name"]) for args in expanded_args_list]
    variables.sort()
    assert variables == [
        (1, "model1"),
        (1, "model2"),
        (2, "model1"),
        (2, "model2"),


        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args)
    assert len(expanded_args_list) == 6

    variables = [(args.n_episode, args.llm_args.model_name) for args in expanded_args_list]
    variables.sort()
    assert variables == [
        (1, "model1"),
        (1, "model2"),
        (2, "model1"),
        (2, "model2"),
        (3, "model1"),
        (3, "model2"),
    ]

def test_cross_product_dict():
    exp_args_dict = dict(
        n_episode=CrossProd([1, 2, 3]),
        llm_args=dict(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args_dict)
    assert len(expanded_args_list) == 6

    variables = [(args["n_episode"], args["llm_args"]["model_name"]) for args in expanded_args_list]
    variables.sort()
    assert variables == [
        (1, "model1"),
        (1, "model2"),
        (2, "model1"),
        (2, "model2"),
        (3, "model1"),
        (3, "model2"),
    ]

def test_sample():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=Choice(["model1", "model2"]),
        ),
    )
    exp_args_sample = sample_args(exp_args, 3)
    assert len(exp_args_sample) == 3
    for exp_args in exp_args_sample:
        assert exp_args.n_episode in [1, 2, 3]
        assert exp_args.llm_args.model_name in ["model1", "model2"]
        assert exp_args.llm_args.temperature == 0.1

def test_sample_and_cross_prod():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),


        llm_args=dict(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    expanded_args_list = expand_cross_product(exp_args_dict)
    assert len(expanded_args_list) == 6

    variables = [(args["n_episode"], args["llm_args"]["model_name"]) for args in expanded_args_list]
    variables.sort()
    assert variables == [
        (1, "model1"),
        (1, "model2"),
        (2, "model1"),
        (2, "model2"),
        (3, "model1"),
        (3, "model2"),
    ]

def test_sample():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=Choice(["model1", "model2"]),
        ),
    )
    exp_args_sample = sample_args(exp_args, 3)
    assert len(exp_args_sample) == 3
    for exp_args in exp_args_sample:
        assert exp_args.n_episode in [1, 2, 3]
        assert exp_args.llm_args.model_name in ["model1", "model2"]
        assert exp_args.llm_args.temperature == 0.1

def test_sample_and_cross_prod():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    def assert_ok(exp_args_list):
        assert len(exp_args_list) == 6
        for exp_args in exp_args_list:
            assert exp_args.n_episode in [1, 2, 3]
            assert exp_args.llm_args.model_name in ["model1", "model2"]
            assert exp_args.llm_args.temperature == 0.1

    exp_args_list_1 = expand_cross_product(sample_args(exp_args, 3))
    assert_ok(exp_args_list_1)
    exp_args_list_2 = sample_args(expand_cross_product(exp_args), 3)


        (3, "model1"),
        (3, "model2"),
    ]

def test_sample():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=Choice(["model1", "model2"]),
        ),
    )
    exp_args_sample = sample_args(exp_args, 3)
    assert len(exp_args_sample) == 3
    for exp_args in exp_args_sample:
        assert exp_args.n_episode in [1, 2, 3]
        assert exp_args.llm_args.model_name in ["model1", "model2"]
        assert exp_args.llm_args.temperature == 0.1

def test_sample_and_cross_prod():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    def assert_ok(exp_args_list):
        assert len(exp_args_list) == 6
        for exp_args in exp_args_list:
            assert exp_args.n_episode in [1, 2, 3]
            assert exp_args.llm_args.model_name in ["model1", "model2"]
            assert exp_args.llm_args.temperature == 0.1

    exp_args_list_1 = expand_cross_product(sample_args(exp_args, 3))
    assert_ok(exp_args_list_1)
    exp_args_list_2 = sample_args(expand_cross_product(exp_args), 3)
    assert_ok(exp_args_list_2)

def test_make_progression_study():
    ablation = make_progression_study(
        start_point=LLMArgsTest(
            model_name="model1",
            temperature=0.1,
        ),
        changes=[
            ("model_name", "model2"),
            ("temperature", 0.2),
        ],
    )

    configs = expand_cross_product(
        ExpArgsTest(
            n_episode=CrossProd([1, 2]),
            llm_args=ablation,
        )
    )


    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    def assert_ok(exp_args_list):
        assert len(exp_args_list) == 6
        for exp_args in exp_args_list:
            assert exp_args.n_episode in [1, 2, 3]
            assert exp_args.llm_args.model_name in ["model1", "model2"]
            assert exp_args.llm_args.temperature == 0.1

    exp_args_list_1 = expand_cross_product(sample_args(exp_args, 3))
    assert_ok(exp_args_list_1)
    exp_args_list_2 = sample_args(expand_cross_product(exp_args), 3)
    assert_ok(exp_args_list_2)

def test_make_progression_study():
    ablation = make_progression_study(
        start_point=LLMArgsTest(
            model_name="model1",
            temperature=0.1,
        ),
        changes=[
            ("model_name", "model2"),
            ("temperature", 0.2),
        ],
    )

    configs = expand_cross_product(
        ExpArgsTest(
            n_episode=CrossProd([1, 2]),
            llm_args=ablation,
        )
    )

    assert len(configs) == 6
    params = [(config.llm_args.model_name, config.llm_args.temperature) for config in configs]
    params = list(set(params))
    params.sort()

    assert params == [("model1", 0.1), ("model2", 0.1), ("model2", 0.2)]

def test_make_ablation_study():
    ablation = make_ablation_study(
        start_point=LLMArgsTest(
            model_name="model1",
            temperature=0.1,
        ),
        changes=[
            ("model_name", "model2"),
            ("temperature", 0.2),
        ],
    )

    configs = expand_cross_product(
        ExpArgsTest(
            n_episode=CrossProd([1, 2]),
            llm_args=ablation,
        )
    )


            ("model_name", "model2"),
            ("temperature", 0.2),
        ],
    )

    configs = expand_cross_product(
        ExpArgsTest(
            n_episode=CrossProd([1, 2]),
            llm_args=ablation,
        )
    )

    assert len(configs) == 6
    params = [(config.llm_args.model_name, config.llm_args.temperature) for config in configs]
    params = list(set(params))
    params.sort()

    assert params == [("model1", 0.1), ("model2", 0.1), ("model2", 0.2)]

def test_make_ablation_study():
    ablation = make_ablation_study(
        start_point=LLMArgsTest(
            model_name="model1",
            temperature=0.1,
        ),
        changes=[
            ("model_name", "model2"),
            ("temperature", 0.2),
        ],
    )

    configs = expand_cross_product(
        ExpArgsTest(
            n_episode=CrossProd([1, 2]),
            llm_args=ablation,
        )
    )

    assert len(configs) == 6
    params = [(config.llm_args.model_name, config.llm_args.temperature) for config in configs]
    params = list(set(params))
    params.sort()

    assert params == [("model1", 0.1), ("model1", 0.2), ("model2", 0.1)]


        llm_args=LLMArgsTest(
            model_name=Choice(["model1", "model2"]),
        ),
    )
    exp_args_sample = sample_args(exp_args, 3)
    assert len(exp_args_sample) == 3
    for exp_args in exp_args_sample:
        assert exp_args.n_episode in [1, 2, 3]
        assert exp_args.llm_args.model_name in ["model1", "model2"]
        assert exp_args.llm_args.temperature == 0.1

def test_sample_and_cross_prod():
    exp_args = ExpArgsTest(
        n_episode=Choice([1, 2, 3]),
        llm_args=LLMArgsTest(
            model_name=CrossProd(["model1", "model2"]),
        ),
    )

    def assert_ok(exp_args_list):
        assert len(exp_args_list) == 6
        for exp_args in exp_args_list:
            assert exp_args.n_episode in [1, 2, 3]
            assert exp_args.llm_args.model_name in ["model1", "model2"]
            assert exp_args.llm_args.temperature == 0.1

    exp_args_list_1 = expand_cross_product(sample_args(exp_args, 3))
    assert_ok(exp_args_list_1)
    exp_args_list_2 = sample_args(expand_cross_product(exp_args), 3)
    assert_ok(exp_args_list_2)

def test_make_progression_study():
    ablation = make_progression_study(
        start_point=LLMArgsTest(
            model_name="model1",
            temperature=0.1,
        ),
        changes=[
            ("model_name", "model2"),
            ("temperature", 0.2),
        ],
    )

    configs = expand_cross_product(


import json
import tempfile
import time
from pathlib import Path

import bgym
import pytest
from bgym import DEFAULT_BENCHMARKS

from agentlab.agents.generic_agent import AGENT_4o_MINI
from agentlab.analyze import inspect_results
from agentlab.experiments import reproducibility_util

@pytest.mark.parametrize(
    "benchmark_name",
    ["miniwob", "workarena_l1", "webarena", "visualwebarena"],
)
def test_get_reproducibility_info(benchmark_name):

    benchmark = DEFAULT_BENCHMARKS[benchmark_name]()

    info = reproducibility_util.get_reproducibility_info(
        "test_agent", benchmark, "test_id", ignore_changes=True
    )

    print("reproducibility info:")
    print(json.dumps(info, indent=4))

    # assert keys in info
    assert "git_user" in info
    assert "benchmark" in info
    assert "benchmark_version" in info
    assert "agentlab_version" in info
    assert "agentlab_git_hash" in info
    assert "agentlab__local_modifications" in info
    assert "browsergym_version" in info
    assert "browsergym_git_hash" in info
    assert "browsergym__local_modifications" in info

# def test_save_reproducibility_info():
#     with tempfile.TemporaryDirectory() as tmp_dir:
#         tmp_dir = Path(tmp_dir)

#         info1 = reproducibility_util.save_reproducibility_info(
#             study_dir=tmp_dir,
#             info=reproducibility_util.get_reproducibility_info(
#                 agents_args="GenericAgent",
#                 benchmark_name="miniwob",
#                 ignore_changes=True,
#             ),
#         )
#         time.sleep(1)  # make sure the date changes by at least 1s

#         # this should overwrite the previous info since they are the same beside
#         # the date
#         info2 = reproducibility_util.save_reproducibility_info(
#             study_dir=tmp_dir,
#             info=reproducibility_util.get_reproducibility_info(
#                 agents_args="GenericAgent",
#                 benchmark_name="miniwob",
#                 ignore_changes=True,
#             ),
#         )

#         reproducibility_util.assert_compatible(info1, info2)

#         # this should not overwrite info2 as the agent name is different, it
#         # should raise an error
#         with pytest.raises(ValueError):
#             reproducibility_util.save_reproducibility_info(
#                 study_dir=tmp_dir,
#                 info=reproducibility_util.get_reproducibility_info(
#                     agents_args="GenericAgent_alt",
#                     benchmark_name="miniwob",
#                     ignore_changes=True,
#                 ),
#             )

#         # load json
#         info3 = reproducibility_util.load_reproducibility_info(tmp_dir)

#         assert info2 == info3
#         assert info1 != info3

#         test_study_dir = Path(__file__).parent.parent / "data" / "test_study"
#         result_df = inspect_results.load_result_df(test_study_dir, progress_fn=None)
#         report_df = inspect_results.summarize_study(result_df)

#         with pytest.raises(ValueError):
#             reproducibility_util.append_to_journal(
#                 info3, report_df, journal_path=tmp_dir / "journal.csv"
#             )

#         reproducibility_util.append_to_journal(
#             info3, report_df, journal_path=tmp_dir / "journal.csv", strict_reproducibility=False
#         )

#         print((tmp_dir / "journal.csv").read_text())

if __name__ == "__main__":
    # test_set_temp()
    test_get_reproducibility_info("miniwob")
    # test_save_reproducibility_info()


import json
import tempfile
import time
from pathlib import Path

import bgym
import pytest
from bgym import DEFAULT_BENCHMARKS

from agentlab.agents.generic_agent import AGENT_4o_MINI
from agentlab.analyze import inspect_results
from agentlab.experiments import reproducibility_util

@pytest.mark.parametrize(
    "benchmark_name",
    ["miniwob", "workarena_l1", "webarena", "visualwebarena"],
)
def test_get_reproducibility_info(benchmark_name):

    benchmark = DEFAULT_BENCHMARKS[benchmark_name]()

    info = reproducibility_util.get_reproducibility_info(
        "test_agent", benchmark, "test_id", ignore_changes=True
    )

    print("reproducibility info:")
    print(json.dumps(info, indent=4))

    # assert keys in info
    assert "git_user" in info
    assert "benchmark" in info
    assert "benchmark_version" in info
    assert "agentlab_version" in info
    assert "agentlab_git_hash" in info
    assert "agentlab__local_modifications" in info
    assert "browsergym_version" in info
    assert "browsergym_git_hash" in info
    assert "browsergym__local_modifications" in info

# def test_save_reproducibility_info():
#     with tempfile.TemporaryDirectory() as tmp_dir:
#         tmp_dir = Path(tmp_dir)

#         info1 = reproducibility_util.save_reproducibility_info(
#             study_dir=tmp_dir,
#             info=reproducibility_util.get_reproducibility_info(
#                 agents_args="GenericAgent",
#                 benchmark_name="miniwob",
#                 ignore_changes=True,
#             ),
#         )
#         time.sleep(1)  # make sure the date changes by at least 1s

#         # this should overwrite the previous info since they are the same beside
#         # the date
#         info2 = reproducibility_util.save_reproducibility_info(
#             study_dir=tmp_dir,


import pytest
from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_4o
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.llm.chat_api import CheatMiniWoBLLMArgs
from agentlab.experiments.study import ParallelStudies, make_study, Study
from agentlab.experiments.multi_server import WebArenaInstanceVars
import logging

logging.getLogger().setLevel(logging.INFO)

def _make_agent_args_list():
    # CheatMiniWoB agents won't succeed on WebArena, this is just for testing parallelization
    agent_args_list = []
    for i in range(2):
        agent_args = GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs(),
            flags=FLAGS_GPT_4o,
        )

        agent_args.agent_name = agent_args.agent_name + f"_{i}"
        agent_args_list.append(agent_args)
    return agent_args_list

@pytest.mark.skip(reason="This test requires WebArena instances to be running")
def manual_test_launch_parallel_study_webarena():
    agent_args_list = _make_agent_args_list()

    server_instance_1 = WebArenaInstanceVars.from_env_vars()
    server_instance_2 = server_instance_1.clone()
    server_instance_2.base_url = "http://webarena-slow.eastus.cloudapp.azure.com"
    parallel_servers = [server_instance_1, server_instance_2]
    # parallel_servers = [server_instance_2]

    for server in parallel_servers:
        print(server)

    study = make_study(
        agent_args_list,
        benchmark="webarena_tiny",
        parallel_servers=parallel_servers,
        ignore_dependencies=True,
    )
    study.override_max_steps(2)
    assert isinstance(study, ParallelStudies)

    study.run(n_jobs=4, parallel_backend="ray", n_relaunch=1)

@pytest.mark.skip(reason="This usecase isnt relevant atm")
def test_launch_parallel_study():
    agent_args_list = _make_agent_args_list()

    study = make_study(agent_args_list, benchmark="miniwob_tiny_test", parallel_servers=2)
    assert isinstance(study, ParallelStudies)

    study.run(n_jobs=4, parallel_backend="ray", n_relaunch=1)
    _, summary_df, _ = study.get_results()
    assert len(summary_df) == 2
    for n_completed in summary_df["n_completed"]:
        assert n_completed == "4/4"

if __name__ == "__main__":
    # test_launch_parallel_study()
    manual_test_launch_parallel_study_webarena()


import pytest
from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_4o
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.llm.chat_api import CheatMiniWoBLLMArgs
from agentlab.experiments.study import ParallelStudies, make_study, Study
from agentlab.experiments.multi_server import WebArenaInstanceVars
import logging

logging.getLogger().setLevel(logging.INFO)

def _make_agent_args_list():
    # CheatMiniWoB agents won't succeed on WebArena, this is just for testing parallelization
    agent_args_list = []
    for i in range(2):
        agent_args = GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs(),
            flags=FLAGS_GPT_4o,
        )

        agent_args.agent_name = agent_args.agent_name + f"_{i}"
        agent_args_list.append(agent_args)
    return agent_args_list

@pytest.mark.skip(reason="This test requires WebArena instances to be running")
def manual_test_launch_parallel_study_webarena():
    agent_args_list = _make_agent_args_list()

    server_instance_1 = WebArenaInstanceVars.from_env_vars()
    server_instance_2 = server_instance_1.clone()
    server_instance_2.base_url = "http://webarena-slow.eastus.cloudapp.azure.com"
    parallel_servers = [server_instance_1, server_instance_2]
    # parallel_servers = [server_instance_2]

    for server in parallel_servers:
        print(server)

    study = make_study(
        agent_args_list,
        benchmark="webarena_tiny",
        parallel_servers=parallel_servers,
        ignore_dependencies=True,


logging.getLogger().setLevel(logging.INFO)

def _make_agent_args_list():
    # CheatMiniWoB agents won't succeed on WebArena, this is just for testing parallelization
    agent_args_list = []
    for i in range(2):
        agent_args = GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs(),
            flags=FLAGS_GPT_4o,
        )

        agent_args.agent_name = agent_args.agent_name + f"_{i}"
        agent_args_list.append(agent_args)
    return agent_args_list

@pytest.mark.skip(reason="This test requires WebArena instances to be running")
def manual_test_launch_parallel_study_webarena():
    agent_args_list = _make_agent_args_list()

    server_instance_1 = WebArenaInstanceVars.from_env_vars()
    server_instance_2 = server_instance_1.clone()
    server_instance_2.base_url = "http://webarena-slow.eastus.cloudapp.azure.com"
    parallel_servers = [server_instance_1, server_instance_2]
    # parallel_servers = [server_instance_2]

    for server in parallel_servers:
        print(server)

    study = make_study(
        agent_args_list,
        benchmark="webarena_tiny",
        parallel_servers=parallel_servers,
        ignore_dependencies=True,
    )
    study.override_max_steps(2)
    assert isinstance(study, ParallelStudies)

    study.run(n_jobs=4, parallel_backend="ray", n_relaunch=1)

@pytest.mark.skip(reason="This usecase isnt relevant atm")
def test_launch_parallel_study():
    agent_args_list = _make_agent_args_list()

    study = make_study(agent_args_list, benchmark="miniwob_tiny_test", parallel_servers=2)
    assert isinstance(study, ParallelStudies)

    study.run(n_jobs=4, parallel_backend="ray", n_relaunch=1)
    _, summary_df, _ = study.get_results()
    assert len(summary_df) == 2
    for n_completed in summary_df["n_completed"]:
        assert n_completed == "4/4"

if __name__ == "__main__":
    # test_launch_parallel_study()
    manual_test_launch_parallel_study_webarena()


    server_instance_2.base_url = "http://webarena-slow.eastus.cloudapp.azure.com"
    parallel_servers = [server_instance_1, server_instance_2]
    # parallel_servers = [server_instance_2]

    for server in parallel_servers:
        print(server)

    study = make_study(
        agent_args_list,
        benchmark="webarena_tiny",
        parallel_servers=parallel_servers,
        ignore_dependencies=True,
    )
    study.override_max_steps(2)
    assert isinstance(study, ParallelStudies)

    study.run(n_jobs=4, parallel_backend="ray", n_relaunch=1)

@pytest.mark.skip(reason="This usecase isnt relevant atm")
def test_launch_parallel_study():
    agent_args_list = _make_agent_args_list()

    study = make_study(agent_args_list, benchmark="miniwob_tiny_test", parallel_servers=2)
    assert isinstance(study, ParallelStudies)

    study.run(n_jobs=4, parallel_backend="ray", n_relaunch=1)
    _, summary_df, _ = study.get_results()
    assert len(summary_df) == 2
    for n_completed in summary_df["n_completed"]:
        assert n_completed == "4/4"

if __name__ == "__main__":
    # test_launch_parallel_study()
    manual_test_launch_parallel_study_webarena()


import importlib.util
import tempfile
from pathlib import Path
from unittest.mock import patch

import pytest

spec = importlib.util.find_spec("desktop_env")
if spec is None:
    DESKTOP_ENV_AVAILABLE = False
    OSWorldActionSet = None
    OsworldEnvArgs = None
    OsworldGym = None
else:
    # If desktop_env is available, import the necessary classes
    from agentlab.benchmarks.osworld import (
        OSWorldActionSet,
        OsworldEnvArgs,
        OsworldGym,
    )

    DESKTOP_ENV_AVAILABLE = True

# Skip the entire module if desktop_env is not available
pytestmark = pytest.mark.skipif(not DESKTOP_ENV_AVAILABLE, reason="desktop_env not installed")

def mock_task_config() -> dict:
    """Mock task configuration for testing."""
    return {
        "id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
        "snapshot": "chrome",
        "instruction": "Can you make Bing the main search thingy when I look stuff up on the internet?",
        "source": "https://support.google.com/chrome/answer/95426",
        "config": [
            {
                "type": "launch",
                "parameters": {"command": ["google-chrome", "--remote-debugging-port=1337"]},
            }
        ],
        "trajectory": "trajectories/",
        "related_apps": ["chrome"],
        "evaluator": {
            "func": "match_in_list",
            "result": {"type": "default_search_engine"},
            "expected": {"type": "rule", "rules": {"expected": ["Microsoft Bing", "Bing"]}},
        },
        "proxy": False,
    }

class TestOSWorldActionSet:
    """Test cases for OSWorld action set functionality."""

    def test_action_set_creation(self):
        """Test basic action set creation."""
        action_set = OSWorldActionSet(action_space="computer_13")
        assert action_set.action_space == "computer_13"

    def test_to_tool_description_openai(self):
        """Test tool description conversion for OpenAI format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="openai")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the expected structure
        tool = tools[0]
        assert "type" in tool
        assert "name" in tool
        assert "description" in tool
        assert "parameters" in tool
        assert tool["type"] == "function"

    def test_to_tool_description_anthropic(self):
        """Test tool description conversion for Anthropic format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="anthropic")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the Anthropic format
        tool = tools[0]
        assert "name" in tool
        assert "description" in tool
        assert "input_schema" in tool
        # Anthropic format doesn't have "type" field

    def test_unsupported_action_space(self):
        """Test that unsupported action spaces raise ValueError."""
        action_set = OSWorldActionSet(action_space="pyautogui")
        with pytest.raises(
            ValueError, match="Only 'computer_13' action space is currently supported"
        ):
            action_set.to_tool_description()

class TestOsworldEnvArgs:
    """Test cases for OSWorld environment arguments."""

    def test_env_args_creation(self):
        """Test basic environment args creation."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task", max_steps=10)

        assert env_args.task == task
        assert env_args.task_name == "test_task"
        assert env_args.max_steps == 10
        assert env_args.action_space == "computer_13"  # default
        assert env_args.provider_name == "docker"  # default

    def test_env_args_custom_config(self):
        """Test environment args with custom configuration."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(
            task=task,
            task_name="custom_task",
            action_space="computer_13",
            provider_name="vmware",
            headless=True,
            screen_size=(1280, 720),
            max_steps=25,
        )

        assert env_args.action_space == "computer_13"
        assert env_args.provider_name == "vmware"
        assert env_args.headless is True
        assert env_args.screen_size == (1280, 720)
        assert env_args.max_steps == 25

    @patch("agentlab.benchmarks.osworld.OsworldGym")
    def test_make_env(self, mock_gym_class):
        """Test environment creation from args."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task")

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            env_args.make_env(exp_dir)

            # Verify that OsworldGym was called with correct arguments
            mock_gym_class.assert_called_once()
            call_args = mock_gym_class.call_args[1]
            assert call_args["task"] == task
            assert call_args["exp_dir"] == exp_dir

class TestOsworldGym:
    """Test cases for OSWorld gym functionality."""

    def test_gym_action_parsing(self):
        """Test gym action parsing functionality."""

        from agentlab.benchmarks.osworld import OsworldGym

        # Test various action strings including edge cases
        test_cases = [
            # Basic actions
            ("wait()", ("wait", [], {})),
            ("done()", ("done", [], {})),
            ("move_to(x=100, y=200)", ("move_to", [], {"x": 100, "y": 200})),
            ('typing(text="hello world")', ("typing", [], {"text": "hello world"})),
            ("hotkey(keys=['ctrl', 'c'])", ("hotkey", [], {"keys": ["ctrl", "c"]})),
            # Edge cases with strings
            ('typing(text="")', ("typing", [], {"text": ""})),  # Empty string
            ('typing(text="line1\\nline2")', ("typing", [], {"text": "line1\nline2"})),  # Newlines
            ('typing(text="tab\\there")', ("typing", [], {"text": "tab\there"})),  # Tabs
            (
                'typing(text="quote\\"test")',
                ("typing", [], {"text": 'quote"test'}),
            ),  # Escaped quotes
            (
                'typing(text="single\'quote")',
                ("typing", [], {"text": "single'quote"}),
            ),  # Single quotes
            ('typing(text="unicode: café")', ("typing", [], {"text": "unicode: café"})),  # Unicode
            # Edge cases with coordinates
            ("move_to(x=0, y=0)", ("move_to", [], {"x": 0, "y": 0})),  # Zero coordinates
            (
                "move_to(x=-10, y=-20)",
                ("move_to", [], {"x": -10, "y": -20}),
            ),  # Negative coordinates
            (
                "move_to(x=9999, y=9999)",
                ("move_to", [], {"x": 9999, "y": 9999}),
            ),  # Large coordinates
            # Edge cases with lists
            ("hotkey(keys=[])", ("hotkey", [], {"keys": []})),  # Empty list
            ("hotkey(keys=['ctrl'])", ("hotkey", [], {"keys": ["ctrl"]})),  # Single key
            (
                "hotkey(keys=['ctrl', 'shift', 'alt', 'a'])",
                ("hotkey", [], {"keys": ["ctrl", "shift", "alt", "a"]}),
            ),  # Multiple keys
            # Edge cases with boolean values
            ("scroll(direction='up', clicks=3)", ("scroll", [], {"direction": "up", "clicks": 3})),
            (
                "click(x=100, y=200, button='left')",
                ("click", [], {"x": 100, "y": 200, "button": "left"}),
            ),
            # Edge cases with mixed parameter types
            (
                "complex_action(text='test', x=50, enabled=True, items=['a', 'b'])",
                (
                    "complex_action",
                    [],
                    {"text": "test", "x": 50, "enabled": True, "items": ["a", "b"]},
                ),
            ),
            # Edge cases with whitespace
            ("  wait()  ", ("wait", [], {})),  # Leading/trailing spaces
            (
                "move_to( x=100 , y=200 )",
                ("move_to", [], {"x": 100, "y": 200}),
            ),  # Spaces around params
            # Edge cases with special characters in strings
            (
                'typing(text="@#$%^&*()+={}[]|\\:;\'<>?,./")',
                ("typing", [], {"text": "@#$%^&*()+={}[]|\\:;'<>?,./"}),
            ),
        ]

        for action_str, expected in test_cases:
            result = OsworldGym.parse_agentlab_action_str_to_func_args(action_str)
            assert result == expected, f"Failed parsing: {action_str}"

    @patch("agentlab.benchmarks.osworld.DesktopEnv")
    def test_gym_creation(self, mock_desktop_env):
        """Test OSWorld gym creation."""
        task = mock_task_config()

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            gym = OsworldGym(
                task=task,
                provider_name="docker",
                region=None,
                path_to_vm=None,
                snapshot_name="init_state",
                action_space="computer_13",
                cache_dir="cache",
                screen_size=(1920, 1080),
                headless=True,
                require_a11y_tree=True,
                require_terminal=False,
                os_type="Ubuntu",
                enable_proxy=False,
                max_steps=50,
                exp_dir=exp_dir,
            )

            assert gym.task == task
            assert gym._step_count == 0
            assert gym.max_steps == 50
            assert gym.exp_dir == exp_dir

    def test_convert_agentlab_action_to_computer_13(self):
        """Test action conversion from AgentLab to Computer 13 format."""
        task = mock_task_config()

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)

            with patch("agentlab.benchmarks.osworld.DesktopEnv"):
                gym = OsworldGym(
                    task=task,
                    provider_name="docker",
                    region=None,
                    path_to_vm=None,
                    snapshot_name="init_state",
                    action_space="computer_13",
                    cache_dir="cache",
                    screen_size=(1920, 1080),
                    headless=True,
                    require_a11y_tree=True,
                    require_terminal=False,
                    os_type="Ubuntu",
                    enable_proxy=False,
                    max_steps=50,
                    exp_dir=exp_dir,
                )

                # Test simple action
                result = gym.convert_agentlab_action_to_computer_13("wait()")
                assert result == "WAIT"

                # Test action with parameters
                result = gym.convert_agentlab_action_to_computer_13("move_to(x=100, y=200)")
                expected = {"action_type": "MOVE_TO", "parameters": {"x": 100, "y": 200}}
                assert result == expected

                # Test typing action
                result = gym.convert_agentlab_action_to_computer_13('typing(text="hello")')
                expected = {"action_type": "TYPING", "parameters": {"text": "hello"}}
                assert result == expected


if spec is None:
    DESKTOP_ENV_AVAILABLE = False
    OSWorldActionSet = None
    OsworldEnvArgs = None
    OsworldGym = None
else:
    # If desktop_env is available, import the necessary classes
    from agentlab.benchmarks.osworld import (
        OSWorldActionSet,
        OsworldEnvArgs,
        OsworldGym,
    )

    DESKTOP_ENV_AVAILABLE = True

# Skip the entire module if desktop_env is not available
pytestmark = pytest.mark.skipif(not DESKTOP_ENV_AVAILABLE, reason="desktop_env not installed")

def mock_task_config() -> dict:
    """Mock task configuration for testing."""
    return {
        "id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
        "snapshot": "chrome",
        "instruction": "Can you make Bing the main search thingy when I look stuff up on the internet?",
        "source": "https://support.google.com/chrome/answer/95426",
        "config": [
            {
                "type": "launch",
                "parameters": {"command": ["google-chrome", "--remote-debugging-port=1337"]},
            }
        ],
        "trajectory": "trajectories/",
        "related_apps": ["chrome"],
        "evaluator": {
            "func": "match_in_list",
            "result": {"type": "default_search_engine"},
            "expected": {"type": "rule", "rules": {"expected": ["Microsoft Bing", "Bing"]}},
        },
        "proxy": False,
    }

class TestOSWorldActionSet:
    """Test cases for OSWorld action set functionality."""

    def test_action_set_creation(self):
        """Test basic action set creation."""
        action_set = OSWorldActionSet(action_space="computer_13")
        assert action_set.action_space == "computer_13"

    def test_to_tool_description_openai(self):
        """Test tool description conversion for OpenAI format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="openai")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the expected structure
        tool = tools[0]


TestOSWorldActionSet

        "snapshot": "chrome",
        "instruction": "Can you make Bing the main search thingy when I look stuff up on the internet?",
        "source": "https://support.google.com/chrome/answer/95426",
        "config": [
            {
                "type": "launch",
                "parameters": {"command": ["google-chrome", "--remote-debugging-port=1337"]},
            }
        ],
        "trajectory": "trajectories/",
        "related_apps": ["chrome"],
        "evaluator": {
            "func": "match_in_list",
            "result": {"type": "default_search_engine"},
            "expected": {"type": "rule", "rules": {"expected": ["Microsoft Bing", "Bing"]}},
        },
        "proxy": False,
    }

class TestOSWorldActionSet:
    """Test cases for OSWorld action set functionality."""

    def test_action_set_creation(self):
        """Test basic action set creation."""
        action_set = OSWorldActionSet(action_space="computer_13")
        assert action_set.action_space == "computer_13"

    def test_to_tool_description_openai(self):
        """Test tool description conversion for OpenAI format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="openai")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the expected structure
        tool = tools[0]
        assert "type" in tool
        assert "name" in tool
        assert "description" in tool
        assert "parameters" in tool
        assert tool["type"] == "function"

    def test_to_tool_description_anthropic(self):
        """Test tool description conversion for Anthropic format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="anthropic")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the Anthropic format
        tool = tools[0]
        assert "name" in tool
        assert "description" in tool
        assert "input_schema" in tool
        # Anthropic format doesn't have "type" field

    def test_unsupported_action_space(self):
        """Test that unsupported action spaces raise ValueError."""
        action_set = OSWorldActionSet(action_space="pyautogui")
        with pytest.raises(
            ValueError, match="Only 'computer_13' action space is currently supported"
        ):
            action_set.to_tool_description()

class TestOsworldEnvArgs:
    """Test cases for OSWorld environment arguments."""

    def test_env_args_creation(self):
        """Test basic environment args creation."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task", max_steps=10)

        assert env_args.task == task
        assert env_args.task_name == "test_task"
        assert env_args.max_steps == 10
        assert env_args.action_space == "computer_13"  # default
        assert env_args.provider_name == "docker"  # default

    def test_env_args_custom_config(self):
        """Test environment args with custom configuration."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(


TestOsworldEnvArgs

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the Anthropic format
        tool = tools[0]
        assert "name" in tool
        assert "description" in tool
        assert "input_schema" in tool
        # Anthropic format doesn't have "type" field

    def test_unsupported_action_space(self):
        """Test that unsupported action spaces raise ValueError."""
        action_set = OSWorldActionSet(action_space="pyautogui")
        with pytest.raises(
            ValueError, match="Only 'computer_13' action space is currently supported"
        ):
            action_set.to_tool_description()

class TestOsworldEnvArgs:
    """Test cases for OSWorld environment arguments."""

    def test_env_args_creation(self):
        """Test basic environment args creation."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task", max_steps=10)

        assert env_args.task == task
        assert env_args.task_name == "test_task"
        assert env_args.max_steps == 10
        assert env_args.action_space == "computer_13"  # default
        assert env_args.provider_name == "docker"  # default

    def test_env_args_custom_config(self):
        """Test environment args with custom configuration."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(
            task=task,
            task_name="custom_task",
            action_space="computer_13",
            provider_name="vmware",
            headless=True,
            screen_size=(1280, 720),
            max_steps=25,
        )

        assert env_args.action_space == "computer_13"
        assert env_args.provider_name == "vmware"
        assert env_args.headless is True
        assert env_args.screen_size == (1280, 720)
        assert env_args.max_steps == 25

    @patch("agentlab.benchmarks.osworld.OsworldGym")
    def test_make_env(self, mock_gym_class):
        """Test environment creation from args."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task")

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            env_args.make_env(exp_dir)

            # Verify that OsworldGym was called with correct arguments
            mock_gym_class.assert_called_once()
            call_args = mock_gym_class.call_args[1]
            assert call_args["task"] == task
            assert call_args["exp_dir"] == exp_dir

class TestOsworldGym:
    """Test cases for OSWorld gym functionality."""

    def test_gym_action_parsing(self):
        """Test gym action parsing functionality."""

        from agentlab.benchmarks.osworld import OsworldGym

        # Test various action strings including edge cases
        test_cases = [
            # Basic actions
            ("wait()", ("wait", [], {})),
            ("done()", ("done", [], {})),
            ("move_to(x=100, y=200)", ("move_to", [], {"x": 100, "y": 200})),
            ('typing(text="hello world")', ("typing", [], {"text": "hello world"})),
            ("hotkey(keys=['ctrl', 'c'])", ("hotkey", [], {"keys": ["ctrl", "c"]})),
            # Edge cases with strings
            ('typing(text="")', ("typing", [], {"text": ""})),  # Empty string


TestOsworldGym

        assert env_args.screen_size == (1280, 720)
        assert env_args.max_steps == 25

    @patch("agentlab.benchmarks.osworld.OsworldGym")
    def test_make_env(self, mock_gym_class):
        """Test environment creation from args."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task")

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            env_args.make_env(exp_dir)

            # Verify that OsworldGym was called with correct arguments
            mock_gym_class.assert_called_once()
            call_args = mock_gym_class.call_args[1]
            assert call_args["task"] == task
            assert call_args["exp_dir"] == exp_dir

class TestOsworldGym:
    """Test cases for OSWorld gym functionality."""

    def test_gym_action_parsing(self):
        """Test gym action parsing functionality."""

        from agentlab.benchmarks.osworld import OsworldGym

        # Test various action strings including edge cases
        test_cases = [
            # Basic actions
            ("wait()", ("wait", [], {})),
            ("done()", ("done", [], {})),
            ("move_to(x=100, y=200)", ("move_to", [], {"x": 100, "y": 200})),
            ('typing(text="hello world")', ("typing", [], {"text": "hello world"})),
            ("hotkey(keys=['ctrl', 'c'])", ("hotkey", [], {"keys": ["ctrl", "c"]})),
            # Edge cases with strings
            ('typing(text="")', ("typing", [], {"text": ""})),  # Empty string
            ('typing(text="line1\\nline2")', ("typing", [], {"text": "line1\nline2"})),  # Newlines
            ('typing(text="tab\\there")', ("typing", [], {"text": "tab\there"})),  # Tabs
            (
                'typing(text="quote\\"test")',
                ("typing", [], {"text": 'quote"test'}),
            ),  # Escaped quotes
            (
                'typing(text="single\'quote")',
                ("typing", [], {"text": "single'quote"}),
            ),  # Single quotes
            ('typing(text="unicode: café")', ("typing", [], {"text": "unicode: café"})),  # Unicode
            # Edge cases with coordinates
            ("move_to(x=0, y=0)", ("move_to", [], {"x": 0, "y": 0})),  # Zero coordinates
            (
                "move_to(x=-10, y=-20)",
                ("move_to", [], {"x": -10, "y": -20}),
            ),  # Negative coordinates
            (
                "move_to(x=9999, y=9999)",
                ("move_to", [], {"x": 9999, "y": 9999}),
            ),  # Large coordinates
            # Edge cases with lists
            ("hotkey(keys=[])", ("hotkey", [], {"keys": []})),  # Empty list
            ("hotkey(keys=['ctrl'])", ("hotkey", [], {"keys": ["ctrl"]})),  # Single key
            (
                "hotkey(keys=['ctrl', 'shift', 'alt', 'a'])",
                ("hotkey", [], {"keys": ["ctrl", "shift", "alt", "a"]}),
            ),  # Multiple keys
            # Edge cases with boolean values
            ("scroll(direction='up', clicks=3)", ("scroll", [], {"direction": "up", "clicks": 3})),
            (
                "click(x=100, y=200, button='left')",
                ("click", [], {"x": 100, "y": 200, "button": "left"}),
            ),
            # Edge cases with mixed parameter types
            (
                "complex_action(text='test', x=50, enabled=True, items=['a', 'b'])",
                (
                    "complex_action",
                    [],
                    {"text": "test", "x": 50, "enabled": True, "items": ["a", "b"]},
                ),
            ),
            # Edge cases with whitespace
            ("  wait()  ", ("wait", [], {})),  # Leading/trailing spaces
            (
                "move_to( x=100 , y=200 )",
                ("move_to", [], {"x": 100, "y": 200}),
            ),  # Spaces around params
            # Edge cases with special characters in strings
            (
                'typing(text="@#$%^&*()+={}[]|\\:;\'<>?,./")',
                ("typing", [], {"text": "@#$%^&*()+={}[]|\\:;'<>?,./"}),
            ),
        ]

        for action_str, expected in test_cases:
            result = OsworldGym.parse_agentlab_action_str_to_func_args(action_str)
            assert result == expected, f"Failed parsing: {action_str}"

    @patch("agentlab.benchmarks.osworld.DesktopEnv")
    def test_gym_creation(self, mock_desktop_env):
        """Test OSWorld gym creation."""
        task = mock_task_config()

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            gym = OsworldGym(
                task=task,
                provider_name="docker",
                region=None,
                path_to_vm=None,
                snapshot_name="init_state",
                action_space="computer_13",
                cache_dir="cache",
                screen_size=(1920, 1080),
                headless=True,
                require_a11y_tree=True,
                require_terminal=False,
                os_type="Ubuntu",
                enable_proxy=False,
                max_steps=50,
                exp_dir=exp_dir,
            )

            assert gym.task == task
            assert gym._step_count == 0
            assert gym.max_steps == 50
            assert gym.exp_dir == exp_dir

    def test_convert_agentlab_action_to_computer_13(self):
        """Test action conversion from AgentLab to Computer 13 format."""
        task = mock_task_config()

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)

            with patch("agentlab.benchmarks.osworld.DesktopEnv"):
                gym = OsworldGym(
                    task=task,
                    provider_name="docker",
                    region=None,
                    path_to_vm=None,
                    snapshot_name="init_state",
                    action_space="computer_13",
                    cache_dir="cache",
                    screen_size=(1920, 1080),
                    headless=True,
                    require_a11y_tree=True,
                    require_terminal=False,
                    os_type="Ubuntu",
                    enable_proxy=False,
                    max_steps=50,
                    exp_dir=exp_dir,
                )

                # Test simple action
                result = gym.convert_agentlab_action_to_computer_13("wait()")
                assert result == "WAIT"

                # Test action with parameters
                result = gym.convert_agentlab_action_to_computer_13("move_to(x=100, y=200)")
                expected = {"action_type": "MOVE_TO", "parameters": {"x": 100, "y": 200}}
                assert result == expected

                # Test typing action
                result = gym.convert_agentlab_action_to_computer_13('typing(text="hello")')
                expected = {"action_type": "TYPING", "parameters": {"text": "hello"}}
                assert result == expected


        "config": [
            {
                "type": "launch",
                "parameters": {"command": ["google-chrome", "--remote-debugging-port=1337"]},
            }
        ],
        "trajectory": "trajectories/",
        "related_apps": ["chrome"],
        "evaluator": {
            "func": "match_in_list",
            "result": {"type": "default_search_engine"},
            "expected": {"type": "rule", "rules": {"expected": ["Microsoft Bing", "Bing"]}},
        },
        "proxy": False,
    }

class TestOSWorldActionSet:
    """Test cases for OSWorld action set functionality."""

    def test_action_set_creation(self):
        """Test basic action set creation."""
        action_set = OSWorldActionSet(action_space="computer_13")
        assert action_set.action_space == "computer_13"

    def test_to_tool_description_openai(self):
        """Test tool description conversion for OpenAI format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="openai")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the expected structure
        tool = tools[0]
        assert "type" in tool
        assert "name" in tool
        assert "description" in tool
        assert "parameters" in tool
        assert tool["type"] == "function"

    def test_to_tool_description_anthropic(self):
        """Test tool description conversion for Anthropic format."""
        action_set = OSWorldActionSet(action_space="computer_13")


        ],
        "trajectory": "trajectories/",
        "related_apps": ["chrome"],
        "evaluator": {
            "func": "match_in_list",
            "result": {"type": "default_search_engine"},
            "expected": {"type": "rule", "rules": {"expected": ["Microsoft Bing", "Bing"]}},
        },
        "proxy": False,
    }

class TestOSWorldActionSet:
    """Test cases for OSWorld action set functionality."""

    def test_action_set_creation(self):
        """Test basic action set creation."""
        action_set = OSWorldActionSet(action_space="computer_13")
        assert action_set.action_space == "computer_13"

    def test_to_tool_description_openai(self):
        """Test tool description conversion for OpenAI format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="openai")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the expected structure
        tool = tools[0]
        assert "type" in tool
        assert "name" in tool
        assert "description" in tool
        assert "parameters" in tool
        assert tool["type"] == "function"

    def test_to_tool_description_anthropic(self):
        """Test tool description conversion for Anthropic format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="anthropic")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the Anthropic format
        tool = tools[0]
        assert "name" in tool
        assert "description" in tool
        assert "input_schema" in tool
        # Anthropic format doesn't have "type" field

    def test_unsupported_action_space(self):
        """Test that unsupported action spaces raise ValueError."""
        action_set = OSWorldActionSet(action_space="pyautogui")
        with pytest.raises(


        """Test basic action set creation."""
        action_set = OSWorldActionSet(action_space="computer_13")
        assert action_set.action_space == "computer_13"

    def test_to_tool_description_openai(self):
        """Test tool description conversion for OpenAI format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="openai")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the expected structure
        tool = tools[0]
        assert "type" in tool
        assert "name" in tool
        assert "description" in tool
        assert "parameters" in tool
        assert tool["type"] == "function"

    def test_to_tool_description_anthropic(self):
        """Test tool description conversion for Anthropic format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="anthropic")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the Anthropic format
        tool = tools[0]
        assert "name" in tool
        assert "description" in tool
        assert "input_schema" in tool
        # Anthropic format doesn't have "type" field

    def test_unsupported_action_space(self):
        """Test that unsupported action spaces raise ValueError."""
        action_set = OSWorldActionSet(action_space="pyautogui")
        with pytest.raises(
            ValueError, match="Only 'computer_13' action space is currently supported"
        ):
            action_set.to_tool_description()

class TestOsworldEnvArgs:
    """Test cases for OSWorld environment arguments."""

    def test_env_args_creation(self):
        """Test basic environment args creation."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task", max_steps=10)

        assert env_args.task == task


        assert "name" in tool
        assert "description" in tool
        assert "parameters" in tool
        assert tool["type"] == "function"

    def test_to_tool_description_anthropic(self):
        """Test tool description conversion for Anthropic format."""
        action_set = OSWorldActionSet(action_space="computer_13")
        tools = action_set.to_tool_description(api="anthropic")

        assert isinstance(tools, list)
        assert len(tools) > 0

        # Check that tools have the Anthropic format
        tool = tools[0]
        assert "name" in tool
        assert "description" in tool
        assert "input_schema" in tool
        # Anthropic format doesn't have "type" field

    def test_unsupported_action_space(self):
        """Test that unsupported action spaces raise ValueError."""
        action_set = OSWorldActionSet(action_space="pyautogui")
        with pytest.raises(
            ValueError, match="Only 'computer_13' action space is currently supported"
        ):
            action_set.to_tool_description()

class TestOsworldEnvArgs:
    """Test cases for OSWorld environment arguments."""

    def test_env_args_creation(self):
        """Test basic environment args creation."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task", max_steps=10)

        assert env_args.task == task
        assert env_args.task_name == "test_task"
        assert env_args.max_steps == 10
        assert env_args.action_space == "computer_13"  # default
        assert env_args.provider_name == "docker"  # default

    def test_env_args_custom_config(self):
        """Test environment args with custom configuration."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(


        # Check that tools have the Anthropic format
        tool = tools[0]
        assert "name" in tool
        assert "description" in tool
        assert "input_schema" in tool
        # Anthropic format doesn't have "type" field

    def test_unsupported_action_space(self):
        """Test that unsupported action spaces raise ValueError."""
        action_set = OSWorldActionSet(action_space="pyautogui")
        with pytest.raises(
            ValueError, match="Only 'computer_13' action space is currently supported"
        ):
            action_set.to_tool_description()

class TestOsworldEnvArgs:
    """Test cases for OSWorld environment arguments."""

    def test_env_args_creation(self):
        """Test basic environment args creation."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task", max_steps=10)

        assert env_args.task == task
        assert env_args.task_name == "test_task"
        assert env_args.max_steps == 10
        assert env_args.action_space == "computer_13"  # default
        assert env_args.provider_name == "docker"  # default

    def test_env_args_custom_config(self):
        """Test environment args with custom configuration."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(
            task=task,
            task_name="custom_task",
            action_space="computer_13",
            provider_name="vmware",
            headless=True,
            screen_size=(1280, 720),
            max_steps=25,
        )

        assert env_args.action_space == "computer_13"
        assert env_args.provider_name == "vmware"
        assert env_args.headless is True
        assert env_args.screen_size == (1280, 720)
        assert env_args.max_steps == 25


        with pytest.raises(
            ValueError, match="Only 'computer_13' action space is currently supported"
        ):
            action_set.to_tool_description()

class TestOsworldEnvArgs:
    """Test cases for OSWorld environment arguments."""

    def test_env_args_creation(self):
        """Test basic environment args creation."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task", max_steps=10)

        assert env_args.task == task
        assert env_args.task_name == "test_task"
        assert env_args.max_steps == 10
        assert env_args.action_space == "computer_13"  # default
        assert env_args.provider_name == "docker"  # default

    def test_env_args_custom_config(self):
        """Test environment args with custom configuration."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(
            task=task,
            task_name="custom_task",
            action_space="computer_13",
            provider_name="vmware",
            headless=True,
            screen_size=(1280, 720),
            max_steps=25,
        )

        assert env_args.action_space == "computer_13"
        assert env_args.provider_name == "vmware"
        assert env_args.headless is True
        assert env_args.screen_size == (1280, 720)
        assert env_args.max_steps == 25

    @patch("agentlab.benchmarks.osworld.OsworldGym")
    def test_make_env(self, mock_gym_class):
        """Test environment creation from args."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task")

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            env_args.make_env(exp_dir)

            # Verify that OsworldGym was called with correct arguments
            mock_gym_class.assert_called_once()
            call_args = mock_gym_class.call_args[1]
            assert call_args["task"] == task
            assert call_args["exp_dir"] == exp_dir

class TestOsworldGym:
    """Test cases for OSWorld gym functionality."""


    def test_env_args_custom_config(self):
        """Test environment args with custom configuration."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(
            task=task,
            task_name="custom_task",
            action_space="computer_13",
            provider_name="vmware",
            headless=True,
            screen_size=(1280, 720),
            max_steps=25,
        )

        assert env_args.action_space == "computer_13"
        assert env_args.provider_name == "vmware"
        assert env_args.headless is True
        assert env_args.screen_size == (1280, 720)
        assert env_args.max_steps == 25

    @patch("agentlab.benchmarks.osworld.OsworldGym")
    def test_make_env(self, mock_gym_class):
        """Test environment creation from args."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task")

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            env_args.make_env(exp_dir)

            # Verify that OsworldGym was called with correct arguments
            mock_gym_class.assert_called_once()
            call_args = mock_gym_class.call_args[1]
            assert call_args["task"] == task
            assert call_args["exp_dir"] == exp_dir

class TestOsworldGym:
    """Test cases for OSWorld gym functionality."""

    def test_gym_action_parsing(self):
        """Test gym action parsing functionality."""

        from agentlab.benchmarks.osworld import OsworldGym

        # Test various action strings including edge cases
        test_cases = [
            # Basic actions
            ("wait()", ("wait", [], {})),
            ("done()", ("done", [], {})),
            ("move_to(x=100, y=200)", ("move_to", [], {"x": 100, "y": 200})),
            ('typing(text="hello world")', ("typing", [], {"text": "hello world"})),
            ("hotkey(keys=['ctrl', 'c'])", ("hotkey", [], {"keys": ["ctrl", "c"]})),
            # Edge cases with strings
            ('typing(text="")', ("typing", [], {"text": ""})),  # Empty string


    @patch("agentlab.benchmarks.osworld.OsworldGym")
    def test_make_env(self, mock_gym_class):
        """Test environment creation from args."""
        task = mock_task_config()
        env_args = OsworldEnvArgs(task=task, task_name="test_task")

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            env_args.make_env(exp_dir)

            # Verify that OsworldGym was called with correct arguments
            mock_gym_class.assert_called_once()
            call_args = mock_gym_class.call_args[1]
            assert call_args["task"] == task
            assert call_args["exp_dir"] == exp_dir

class TestOsworldGym:
    """Test cases for OSWorld gym functionality."""

    def test_gym_action_parsing(self):
        """Test gym action parsing functionality."""

        from agentlab.benchmarks.osworld import OsworldGym

        # Test various action strings including edge cases
        test_cases = [
            # Basic actions
            ("wait()", ("wait", [], {})),
            ("done()", ("done", [], {})),
            ("move_to(x=100, y=200)", ("move_to", [], {"x": 100, "y": 200})),
            ('typing(text="hello world")', ("typing", [], {"text": "hello world"})),
            ("hotkey(keys=['ctrl', 'c'])", ("hotkey", [], {"keys": ["ctrl", "c"]})),
            # Edge cases with strings
            ('typing(text="")', ("typing", [], {"text": ""})),  # Empty string
            ('typing(text="line1\\nline2")', ("typing", [], {"text": "line1\nline2"})),  # Newlines
            ('typing(text="tab\\there")', ("typing", [], {"text": "tab\there"})),  # Tabs
            (
                'typing(text="quote\\"test")',
                ("typing", [], {"text": 'quote"test'}),
            ),  # Escaped quotes
            (
                'typing(text="single\'quote")',
                ("typing", [], {"text": "single'quote"}),
            ),  # Single quotes
            ('typing(text="unicode: café")', ("typing", [], {"text": "unicode: café"})),  # Unicode
            # Edge cases with coordinates
            ("move_to(x=0, y=0)", ("move_to", [], {"x": 0, "y": 0})),  # Zero coordinates
            (
                "move_to(x=-10, y=-20)",
                ("move_to", [], {"x": -10, "y": -20}),
            ),  # Negative coordinates
            (
                "move_to(x=9999, y=9999)",
                ("move_to", [], {"x": 9999, "y": 9999}),
            ),  # Large coordinates
            # Edge cases with lists
            ("hotkey(keys=[])", ("hotkey", [], {"keys": []})),  # Empty list
            ("hotkey(keys=['ctrl'])", ("hotkey", [], {"keys": ["ctrl"]})),  # Single key
            (
                "hotkey(keys=['ctrl', 'shift', 'alt', 'a'])",
                ("hotkey", [], {"keys": ["ctrl", "shift", "alt", "a"]}),
            ),  # Multiple keys
            # Edge cases with boolean values
            ("scroll(direction='up', clicks=3)", ("scroll", [], {"direction": "up", "clicks": 3})),
            (
                "click(x=100, y=200, button='left')",
                ("click", [], {"x": 100, "y": 200, "button": "left"}),
            ),
            # Edge cases with mixed parameter types
            (
                "complex_action(text='test', x=50, enabled=True, items=['a', 'b'])",
                (
                    "complex_action",
                    [],
                    {"text": "test", "x": 50, "enabled": True, "items": ["a", "b"]},
                ),
            ),
            # Edge cases with whitespace
            ("  wait()  ", ("wait", [], {})),  # Leading/trailing spaces
            (
                "move_to( x=100 , y=200 )",
                ("move_to", [], {"x": 100, "y": 200}),
            ),  # Spaces around params
            # Edge cases with special characters in strings
            (
                'typing(text="@#$%^&*()+={}[]|\\:;\'<>?,./")',
                ("typing", [], {"text": "@#$%^&*()+={}[]|\\:;'<>?,./"}),
            ),
        ]

        for action_str, expected in test_cases:
            result = OsworldGym.parse_agentlab_action_str_to_func_args(action_str)
            assert result == expected, f"Failed parsing: {action_str}"

    @patch("agentlab.benchmarks.osworld.DesktopEnv")
    def test_gym_creation(self, mock_desktop_env):
        """Test OSWorld gym creation."""
        task = mock_task_config()

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            gym = OsworldGym(
                task=task,
                provider_name="docker",
                region=None,
                path_to_vm=None,
                snapshot_name="init_state",
                action_space="computer_13",
                cache_dir="cache",
                screen_size=(1920, 1080),
                headless=True,
                require_a11y_tree=True,
                require_terminal=False,


                ),
            ),
            # Edge cases with whitespace
            ("  wait()  ", ("wait", [], {})),  # Leading/trailing spaces
            (
                "move_to( x=100 , y=200 )",
                ("move_to", [], {"x": 100, "y": 200}),
            ),  # Spaces around params
            # Edge cases with special characters in strings
            (
                'typing(text="@#$%^&*()+={}[]|\\:;\'<>?,./")',
                ("typing", [], {"text": "@#$%^&*()+={}[]|\\:;'<>?,./"}),
            ),
        ]

        for action_str, expected in test_cases:
            result = OsworldGym.parse_agentlab_action_str_to_func_args(action_str)
            assert result == expected, f"Failed parsing: {action_str}"

    @patch("agentlab.benchmarks.osworld.DesktopEnv")
    def test_gym_creation(self, mock_desktop_env):
        """Test OSWorld gym creation."""
        task = mock_task_config()

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)
            gym = OsworldGym(
                task=task,
                provider_name="docker",
                region=None,
                path_to_vm=None,
                snapshot_name="init_state",
                action_space="computer_13",
                cache_dir="cache",
                screen_size=(1920, 1080),
                headless=True,
                require_a11y_tree=True,
                require_terminal=False,
                os_type="Ubuntu",
                enable_proxy=False,
                max_steps=50,
                exp_dir=exp_dir,
            )

            assert gym.task == task
            assert gym._step_count == 0
            assert gym.max_steps == 50
            assert gym.exp_dir == exp_dir

    def test_convert_agentlab_action_to_computer_13(self):
        """Test action conversion from AgentLab to Computer 13 format."""
        task = mock_task_config()

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)

            with patch("agentlab.benchmarks.osworld.DesktopEnv"):
                gym = OsworldGym(
                    task=task,
                    provider_name="docker",
                    region=None,
                    path_to_vm=None,
                    snapshot_name="init_state",
                    action_space="computer_13",
                    cache_dir="cache",
                    screen_size=(1920, 1080),
                    headless=True,
                    require_a11y_tree=True,


                region=None,
                path_to_vm=None,
                snapshot_name="init_state",
                action_space="computer_13",
                cache_dir="cache",
                screen_size=(1920, 1080),
                headless=True,
                require_a11y_tree=True,
                require_terminal=False,
                os_type="Ubuntu",
                enable_proxy=False,
                max_steps=50,
                exp_dir=exp_dir,
            )

            assert gym.task == task
            assert gym._step_count == 0
            assert gym.max_steps == 50
            assert gym.exp_dir == exp_dir

    def test_convert_agentlab_action_to_computer_13(self):
        """Test action conversion from AgentLab to Computer 13 format."""
        task = mock_task_config()

        with tempfile.TemporaryDirectory() as tmp_dir:
            exp_dir = Path(tmp_dir)

            with patch("agentlab.benchmarks.osworld.DesktopEnv"):
                gym = OsworldGym(
                    task=task,
                    provider_name="docker",
                    region=None,
                    path_to_vm=None,
                    snapshot_name="init_state",
                    action_space="computer_13",
                    cache_dir="cache",
                    screen_size=(1920, 1080),
                    headless=True,
                    require_a11y_tree=True,
                    require_terminal=False,
                    os_type="Ubuntu",
                    enable_proxy=False,
                    max_steps=50,
                    exp_dir=exp_dir,
                )

                # Test simple action
                result = gym.convert_agentlab_action_to_computer_13("wait()")
                assert result == "WAIT"

                # Test action with parameters
                result = gym.convert_agentlab_action_to_computer_13("move_to(x=100, y=200)")
                expected = {"action_type": "MOVE_TO", "parameters": {"x": 100, "y": 200}}
                assert result == expected

                # Test typing action
                result = gym.convert_agentlab_action_to_computer_13('typing(text="hello")')
                expected = {"action_type": "TYPING", "parameters": {"text": "hello"}}
                assert result == expected


import re
import tempfile
from dataclasses import dataclass
from pathlib import Path

from openai import OpenAIError

from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_3_5
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.analyze import inspect_results
from agentlab.experiments import launch_exp
from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.llm.chat_api import BaseModelArgs, CheatMiniWoBLLMArgs
from agentlab.llm.llm_utils import Discussion

def test_generic_agent():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs(),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )

        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "n_steps": 1,
            "cum_reward": 1.0,
            "terminated": True,
            "truncated": False,
            "err_msg": None,
            "stack_trace": None,
            "agent.flags.obs.use_ax_tree": True,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

@dataclass
class CheatMiniWoBLLM_ParseRetry:
    """For unit-testing purposes only. It only work with miniwob.click-test task."""

    n_retry: int
    retry_count: int = 0

    def __call__(self, messages) -> str:
        if self.retry_count < self.n_retry:
            self.retry_count += 1
            return dict(role="assistant", content="I'm retrying")

        if isinstance(messages, Discussion):
            prompt = messages.to_string()
        else:
            prompt = messages[1].get("content", "")
        match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)

        if match:
            bid = match.group(1)
            action = f'click("{bid}")'
        else:
            raise Exception("Can't find the button's bid")

        answer = f"""I'm clicking the button as requested.
<action>
{action}
</action>
"""
        return dict(role="assistant", content=answer)

    def get_stats(self):
        return {}

@dataclass
class CheatMiniWoBLLMArgs_ParseRetry(BaseModelArgs):
    n_retry: int = 2
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatMiniWoBLLM_ParseRetry(n_retry=self.n_retry)

@dataclass
class CheatLLM_LLMError:
    """For unit-testing purposes only. Fails to call LLM"""

    n_retry: int = 0
    success: bool = False

    def __call__(self, messages) -> str:
        if self.success:
            if isinstance(messages, Discussion):
                prompt = messages.to_string()
            else:
                prompt = messages[1].get("content", "")
            match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)

            if match:
                bid = match.group(1)
                action = f'click("{bid}")'
            else:
                raise Exception("Can't find the button's bid")

            answer = f"""I'm clicking the button as requested.
    <action>
    {action}
    </action>
    """
            return dict(role="assistant", content=answer)
        raise OpenAIError("LLM failed to respond")

    def get_stats(self):
        return {"n_llm_retry": self.n_retry, "n_llm_busted_retry": int(not self.success)}

@dataclass
class CheatLLMArgs_LLMError(BaseModelArgs):
    n_retry: int = 2
    success: bool = False
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatLLM_LLMError(
            n_retry=self.n_retry,
            success=self.success,
        )

def test_generic_agent_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=2),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        # TODO why these tests don't work with ray backend?
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)
        print(result_record)
        target = {
            "stats.cum_n_retry": 2,
            "stats.cum_busted_retry": 0,
            "n_steps": 1,
            "cum_reward": 1.0,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_bust_parse_retry():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs_ParseRetry(n_retry=10),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "stats.cum_n_retry": 5,
            "stats.cum_busted_retry": 1,
            "n_steps": 0,
            "cum_reward": 0,
            "err_msg": None,  # parsing error is considered an agent failure, not a code error
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_llm_error_success():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatLLMArgs_LLMError(n_retry=3, success=True),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "stats.cum_n_llm_retry": 3,
            "n_steps": 1,
            "cum_reward": 1.0,
            "err_msg": None,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

def test_llm_error_no_success():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatLLMArgs_LLMError(n_retry=5, success=False),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )
        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "n_steps": 0,
            "cum_reward": 0,
            "err_msg": "Exception uncaught by agent or environment in task miniwob.click-test.\nOpenAIError:\nLLM failed to respond",
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

if __name__ == "__main__":
    # test_generic_agent()
    test_generic_agent_parse_retry()


import re
import tempfile
from dataclasses import dataclass
from pathlib import Path

from openai import OpenAIError

from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_3_5
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.analyze import inspect_results
from agentlab.experiments import launch_exp
from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.llm.chat_api import BaseModelArgs, CheatMiniWoBLLMArgs
from agentlab.llm.llm_utils import Discussion

def test_generic_agent():
    exp_args = ExpArgs(
        agent_args=GenericAgentArgs(
            chat_model_args=CheatMiniWoBLLMArgs(),
            flags=FLAGS_GPT_3_5,
        ),
        env_args=EnvArgs(task_name="miniwob.click-test", task_seed=42),
    )

    with tempfile.TemporaryDirectory() as tmp_dir:
        launch_exp.run_experiments(
            1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
        )

        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "n_steps": 1,
            "cum_reward": 1.0,
            "terminated": True,
            "truncated": False,
            "err_msg": None,
            "stack_trace": None,
            "agent.flags.obs.use_ax_tree": True,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

@dataclass
class CheatMiniWoBLLM_ParseRetry:
    """For unit-testing purposes only. It only work with miniwob.click-test task."""

    n_retry: int
    retry_count: int = 0

    def __call__(self, messages) -> str:
        if self.retry_count < self.n_retry:
            self.retry_count += 1
            return dict(role="assistant", content="I'm retrying")

        if isinstance(messages, Discussion):
            prompt = messages.to_string()
        else:
            prompt = messages[1].get("content", "")
        match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)


        )

        result_record = inspect_results.load_result_df(tmp_dir, progress_fn=None)

        target = {
            "n_steps": 1,
            "cum_reward": 1.0,
            "terminated": True,
            "truncated": False,
            "err_msg": None,
            "stack_trace": None,
            "agent.flags.obs.use_ax_tree": True,
        }

        for key, target_val in target.items():
            assert key in result_record
            assert result_record[key].iloc[0] == target_val

@dataclass
class CheatMiniWoBLLM_ParseRetry:
    """For unit-testing purposes only. It only work with miniwob.click-test task."""

    n_retry: int
    retry_count: int = 0

    def __call__(self, messages) -> str:
        if self.retry_count < self.n_retry:
            self.retry_count += 1
            return dict(role="assistant", content="I'm retrying")

        if isinstance(messages, Discussion):
            prompt = messages.to_string()
        else:
            prompt = messages[1].get("content", "")
        match = re.search(r"^\s*\[(\d+)\].*button", prompt, re.MULTILINE | re.IGNORECASE)

        if match:
            bid = match.group(1)
            action = f'click("{bid}")'
        else:
            raise Exception("Can't find the button's bid")

        answer = f"""I'm clicking the button as requested.
<action>
{action}
</action>
"""
        return dict(role="assistant", content=answer)

    def get_stats(self):
        return {}

@dataclass
class CheatMiniWoBLLMArgs_ParseRetry(BaseModelArgs):
    n_retry: int = 2
    model_name: str = "test/cheat_miniwob_click_test_parse_retry"

    def make_model(self):
        return CheatMiniWoBLLM_ParseRetry(n_retry=self.n_retry)

@dataclass
class CheatLLM_LLMError:
    """For unit-testing purposes only. Fails to call LLM"""

    n_retry: int = 0
    success: bool = False

    def __call__(self, messages) -> str:
        if self.success: