        self.make_stats()

        return self.action

    def from_reset(self, env: gym.Env, seed: int, obs_preprocessor: callable):
        t = self.profiling
        t.env_start = time.time()
        self.obs, env_info = env.reset(seed=seed)
        self.reward, self.terminated, self.truncated = 0, False, False
        t.env_stop = time.time()

        t.action_exec_start = env_info.get("recording_start_time", t.env_start)
        t.action_exect_after_timeout = t.env_stop
        t.action_exec_stop = t.env_stop

        if obs_preprocessor:
            self.obs = obs_preprocessor(self.obs)

    @property
    def is_done(self):
        return self.terminated or self.truncated

    def make_stats(self):
        if isinstance(self.obs, dict):
            stats = {
                f"n_token_{key}": count_tokens(val)
                for key, val in self.obs.items()
                if isinstance(val, str)
            }
        else:
            stats = {}
        stats.update(self.agent_info.pop("stats", {}))

        t = self.profiling
        stats["step_elapsed"] = t.env_stop - t.env_start
        stats["agent_elapsed"] = t.agent_stop - t.agent_start

        self.stats = stats

    def save_step_info(self, exp_dir, save_json=False, save_screenshot=True, save_som=False):
        # special treatment for some of the observation fields


        return self.action

    def from_reset(self, env: gym.Env, seed: int, obs_preprocessor: callable):
        t = self.profiling
        t.env_start = time.time()
        self.obs, env_info = env.reset(seed=seed)
        self.reward, self.terminated, self.truncated = 0, False, False
        t.env_stop = time.time()

        t.action_exec_start = env_info.get("recording_start_time", t.env_start)
        t.action_exect_after_timeout = t.env_stop
        t.action_exec_stop = t.env_stop

        if obs_preprocessor:
            self.obs = obs_preprocessor(self.obs)

    @property
    def is_done(self):
        return self.terminated or self.truncated

    def make_stats(self):
        if isinstance(self.obs, dict):
            stats = {
                f"n_token_{key}": count_tokens(val)
                for key, val in self.obs.items()
                if isinstance(val, str)
            }
        else:
            stats = {}
        stats.update(self.agent_info.pop("stats", {}))

        t = self.profiling
        stats["step_elapsed"] = t.env_stop - t.env_start
        stats["agent_elapsed"] = t.agent_stop - t.agent_start

        self.stats = stats

    def save_step_info(self, exp_dir, save_json=False, save_screenshot=True, save_som=False):
        # special treatment for some of the observation fields
        if isinstance(self.obs, dict):
            # save screenshots to separate files
            screenshot = self.obs.pop("screenshot", None)
            screenshot_som = self.obs.pop("screenshot_som", None)

            if save_screenshot and screenshot is not None:
                img = Image.fromarray(screenshot)
                img.save(exp_dir / f"screenshot_step_{self.step}.png")

            if save_som and screenshot_som is not None:
                img = Image.fromarray(screenshot_som)
                img.save(exp_dir / f"screenshot_som_step_{self.step}.png")

            # save goal object (which might contain images) to a separate file to save space
            if self.obs.get("goal_object", False):
                # save the goal object only once (goal should never change once setup)
                goal_object_file = Path(exp_dir) / "goal_object.pkl.gz"


    def is_done(self):
        return self.terminated or self.truncated

    def make_stats(self):
        if isinstance(self.obs, dict):
            stats = {
                f"n_token_{key}": count_tokens(val)
                for key, val in self.obs.items()
                if isinstance(val, str)
            }
        else:
            stats = {}
        stats.update(self.agent_info.pop("stats", {}))

        t = self.profiling
        stats["step_elapsed"] = t.env_stop - t.env_start
        stats["agent_elapsed"] = t.agent_stop - t.agent_start

        self.stats = stats

    def save_step_info(self, exp_dir, save_json=False, save_screenshot=True, save_som=False):
        # special treatment for some of the observation fields
        if isinstance(self.obs, dict):
            # save screenshots to separate files
            screenshot = self.obs.pop("screenshot", None)
            screenshot_som = self.obs.pop("screenshot_som", None)

            if save_screenshot and screenshot is not None:
                img = Image.fromarray(screenshot)
                img.save(exp_dir / f"screenshot_step_{self.step}.png")

            if save_som and screenshot_som is not None:
                img = Image.fromarray(screenshot_som)
                img.save(exp_dir / f"screenshot_som_step_{self.step}.png")

            # save goal object (which might contain images) to a separate file to save space
            if self.obs.get("goal_object", False):
                # save the goal object only once (goal should never change once setup)
                goal_object_file = Path(exp_dir) / "goal_object.pkl.gz"
                if not goal_object_file.exists():
                    with gzip.open(goal_object_file, "wb") as f:
                        pickle.dump(self.obs["goal_object"], f)
                # set goal_object to a special placeholder value, which indicates it should be loaded from a separate file
                self.obs["goal_object"] = None

        with gzip.open(exp_dir / f"step_{self.step}.pkl.gz", "wb") as f:
            pickle.dump(self, f)

        if save_json:
            with open(exp_dir / "steps_info.json", "w") as f:
                json.dump(self, f, indent=4, cls=DataclassJSONEncoder)

        if isinstance(self.obs, dict):
            # add the screenshots back to the obs
            # why do we need this?
            if screenshot is not None:
                self.obs["screenshot"] = screenshot
            if screenshot_som is not None:
                self.obs["screenshot_som"] = screenshot_som

@dataclass
class ExpArgs:
    """Arguments to run an experiment, i.e. run agent in an environment until done.

    This dataclass is used to store experiments arguments. It contains
    agent_args and env_args which follows the same principle. It contains helper
    functions to prepare and run experiments.

    Attributes:
    -----------
    agent_args: AbstractAgentArgs
        The arguments to instantiate the agent.
    env_args: EnvArgs
        The arguments to instantiate the environment.
    exp_dir: str
        The directory where the experiment will be saved.
    exp_name: str
        The name of the experiment. If None, it will be generated from the


    order: int (internal)
        The order of the experiment in the batch. It is used to keep track of
        the original order of the experiments in case they are shuffled.
    """

    agent_args: AbstractAgentArgs
    env_args: EnvArgs
    exp_dir: str = None
    exp_name: str = None
    enable_debug: bool = True
    err_msg: str = None
    stack_trace: str = None
    order: int = None  # use to keep the original order the experiments were meant to be launched.
    logging_level: int = logging.INFO
    logging_level_stdout: int = logging.INFO
    exp_id: str = None
    depends_on: tuple[str] = ()
    save_screenshot: bool = True
    save_som: bool = False

    def make_id(self):
        """Create a unique id for the experiment."""
        if self.exp_id is None:
            self.exp_id = str(uuid.uuid4())

    def prepare(self, exp_root):
        """Prepare the experiment directory and save the experiment arguments.

        This enables inspecting experiments that are not run yet.

        Args:
            exp_root: str
                The root directory where the experiment will be saved.
        """
        if self.env_args.task_seed is None:
            self.env_args.task_seed = np.random.randint(0, SEED_MAX)

        if self.exp_name is None:
            task_name = self.env_args.task_name
            self.exp_name = f"{self.agent_args.agent_name}_on_{task_name}_{self.env_args.task_seed}"

        # if exp_dir exists, it means it's a re-run, move the old one
        if self.exp_dir is not None:
            _move_old_exp(self.exp_dir)


        if self.env_args.task_seed is None:
            self.env_args.task_seed = np.random.randint(0, SEED_MAX)

        if self.exp_name is None:
            task_name = self.env_args.task_name
            self.exp_name = f"{self.agent_args.agent_name}_on_{task_name}_{self.env_args.task_seed}"

        # if exp_dir exists, it means it's a re-run, move the old one
        if self.exp_dir is not None:
            _move_old_exp(self.exp_dir)

        self.make_id()

        self.exp_date = datetime.now()
        self._make_dir(exp_root)

        self.exp_dir.mkdir(parents=True, exist_ok=True)
        with open(self.exp_dir / "exp_args.pkl", "wb") as f:
            pickle.dump(self, f)

    def _make_dir(self, exp_root):
        """Create a unique directory for the experiment."""
        date_str = self.exp_date.strftime("%Y-%m-%d_%H-%M-%S")
        exp_str = re.sub(
            r"[\/:*?<>|]", "_", self.exp_name
        )  # sanitize exp_name to be used as a file name (substitute forbidden characters)

        for i in range(1000):
            if i >= 999:  # make sure we don't loop forever
                raise ValueError("Could not find a unique name for the experiment directory.")

            tag = f"_{i}" if i > 0 else ""
            self.exp_dir = Path(exp_root) / f"{date_str}_{exp_str}{tag}"
            if not self.exp_dir.exists():
                break

    # TODO distinguish between agent error and environment or system error. e.g.
    # the parsing error of an action should not be re-run.
    def run(self):
        """Run the experiment and save the results"""
        # start writing logs to run logfile
        self._set_logger()

        # log python environment info
        save_package_versions(Path(self.exp_dir))

        episode_info = []
        agent = None
        env, step_info, err_msg, stack_trace = None, None, None, None
        try:
            logger.info(f"Running experiment {self.exp_name} in:\n  {self.exp_dir}")
            agent = self.agent_args.make_agent()
            if hasattr(agent, "set_task_name"):
                agent.set_task_name(self.env_args.task_name)


run

            pickle.dump(self, f)

    def _make_dir(self, exp_root):
        """Create a unique directory for the experiment."""
        date_str = self.exp_date.strftime("%Y-%m-%d_%H-%M-%S")
        exp_str = re.sub(
            r"[\/:*?<>|]", "_", self.exp_name
        )  # sanitize exp_name to be used as a file name (substitute forbidden characters)

        for i in range(1000):
            if i >= 999:  # make sure we don't loop forever
                raise ValueError("Could not find a unique name for the experiment directory.")

            tag = f"_{i}" if i > 0 else ""
            self.exp_dir = Path(exp_root) / f"{date_str}_{exp_str}{tag}"
            if not self.exp_dir.exists():
                break

    # TODO distinguish between agent error and environment or system error. e.g.
    # the parsing error of an action should not be re-run.
    def run(self):
        """Run the experiment and save the results"""
        # start writing logs to run logfile
        self._set_logger()

        # log python environment info
        save_package_versions(Path(self.exp_dir))

        episode_info = []
        agent = None
        env, step_info, err_msg, stack_trace = None, None, None, None
        try:
            logger.info(f"Running experiment {self.exp_name} in:\n  {self.exp_dir}")
            agent = self.agent_args.make_agent()
            if hasattr(agent, "set_task_name"):
                agent.set_task_name(self.env_args.task_name)

            logger.debug("Agent created.")

            env = self.env_args.make_env(
                action_mapping=agent.action_set.to_python_code,
                exp_dir=self.exp_dir,
                use_raw_page_output=getattr(self.agent_args, "use_raw_page_output", False),
            )

            logger.debug("Environment created.")
            step_info = StepInfo(step=0)
            episode_info = [step_info]
            step_info.from_reset(
                env, seed=self.env_args.task_seed or 0, obs_preprocessor=agent.obs_preprocessor
            )
            logger.debug("Environment reset.")

            while not step_info.is_done:  # set a limit
                logger.debug(f"Starting step {step_info.step}.")
                action = step_info.from_action(agent)
                logger.debug(f"Agent chose action:\n {action}")

                if action is None:
                    # will end the episode after saving the step info.
                    step_info.truncated = True

                step_info.save_step_info(
                    self.exp_dir, save_screenshot=self.save_screenshot, save_som=self.save_som
                )
                logger.debug("Step info saved.")

                if hasattr(env.unwrapped, "chat") and isinstance(env.unwrapped.chat, Chat):
                    _send_chat_info(env.unwrapped.chat, action, step_info.agent_info)
                    logger.debug("Chat info sent.")

                if action is None:
                    logger.debug("Agent returned None action. Ending episode.")
                    break

                step_info = StepInfo(step=step_info.step + 1)
                episode_info.append(step_info)

                logger.debug("Sending action to environment.")
                step_info.from_step(env, action, obs_preprocessor=agent.obs_preprocessor)
                logger.debug("Environment stepped.")
                if step_info.is_done:
                    logger.debug(
                        f"Episode done: terminated: {step_info.terminated}, truncated: {step_info.truncated}."
                    )

        except Exception as e:
            err_msg = f"Exception uncaught by agent or environment in task {self.env_args.task_name}.\n{type(e).__name__}:\n{e}"
            stack_trace = traceback.format_exc()

            self.err_msg = err_msg
            self.stack_trace = stack_trace

            logger.warning(err_msg + "\n" + stack_trace)
            if _is_debugging() and self.enable_debug:
                logger.warning("Debug mode is enabled. Raising the error.")
                raise

        finally:
            try:
                if step_info is not None:
                    step_info.save_step_info(
                        self.exp_dir, save_screenshot=self.save_screenshot, save_som=self.save_som
                    )
            except Exception as e:
                logger.error(f"Error while saving step info in the finally block: {e}")
            try:
                if (
                    not err_msg
                    and len(episode_info) > 0
                    and not (episode_info[-1].terminated or episode_info[-1].truncated)
                ):
                    e = KeyboardInterrupt("Early termination??")
                    err_msg = f"Exception uncaught by agent or environment in task {self.env_args.task_name}.\n{type(e).__name__}:\n{e}"
                logger.info("Saving experiment info.")
                self.save_summary_info(episode_info, Path(self.exp_dir), err_msg, stack_trace)
                if TapeAgent is not None and isinstance(agent, TapeAgent):
                    task = getattr(env, "task", {})
                    save_tape(self.exp_dir, episode_info, task, agent.final_tape)
            except Exception as e:
                logger.exception(f"Error while saving experiment info: {e}")
            try:
                if env is not None:
                    env.close()
            except Exception as e:
                logger.exception(f"Error while closing the environment: {e}")
            try:
                self._unset_logger()  # stop writing logs to run logfile
            except Exception as e:
                logger.exception(f"Error while unsetting the logger: {e}")

    def _set_logger(self):
        # output logging traces to a log file
        file_handler = logging.FileHandler(self.exp_dir / "experiment.log")
        file_handler.setLevel(self.logging_level)  # same level as console outputs
        formatter = logging.Formatter(
            "%(asctime)s - %(process)d - %(name)s - %(levelname)s - %(message)s"
        )
        file_handler.setFormatter(formatter)
        # output handler
        stream_handler = logging.StreamHandler()
        stream_handler.setLevel(self.logging_level_stdout)
        stream_handler.setFormatter(formatter)
        # setup root logger
        root_logger = logging.getLogger()

        # remove previous stream handlers
        for handler in root_logger.handlers:
            if isinstance(handler, logging.StreamHandler):
                root_logger.removeHandler(handler)


                ):
                    e = KeyboardInterrupt("Early termination??")
                    err_msg = f"Exception uncaught by agent or environment in task {self.env_args.task_name}.\n{type(e).__name__}:\n{e}"
                logger.info("Saving experiment info.")
                self.save_summary_info(episode_info, Path(self.exp_dir), err_msg, stack_trace)
                if TapeAgent is not None and isinstance(agent, TapeAgent):
                    task = getattr(env, "task", {})
                    save_tape(self.exp_dir, episode_info, task, agent.final_tape)
            except Exception as e:
                logger.exception(f"Error while saving experiment info: {e}")
            try:
                if env is not None:
                    env.close()
            except Exception as e:
                logger.exception(f"Error while closing the environment: {e}")
            try:
                self._unset_logger()  # stop writing logs to run logfile
            except Exception as e:
                logger.exception(f"Error while unsetting the logger: {e}")

    def _set_logger(self):
        # output logging traces to a log file
        file_handler = logging.FileHandler(self.exp_dir / "experiment.log")
        file_handler.setLevel(self.logging_level)  # same level as console outputs
        formatter = logging.Formatter(
            "%(asctime)s - %(process)d - %(name)s - %(levelname)s - %(message)s"
        )
        file_handler.setFormatter(formatter)
        # output handler
        stream_handler = logging.StreamHandler()
        stream_handler.setLevel(self.logging_level_stdout)
        stream_handler.setFormatter(formatter)
        # setup root logger
        root_logger = logging.getLogger()

        # remove previous stream handlers
        for handler in root_logger.handlers:
            if isinstance(handler, logging.StreamHandler):
                root_logger.removeHandler(handler)

        root_logger.setLevel(self.logging_level)
        root_logger.addHandler(file_handler)
        root_logger.addHandler(stream_handler)
        # setup openai logger (don't go below INFO verbosity)
        openai_logger = logging.getLogger("openai._base_client")
        openai_logger.setLevel(max(logging.INFO, self.logging_level))

        self.logging_file_handler = file_handler

    def _unset_logger(self):
        root_logger = logging.getLogger()
        root_logger.removeHandler(self.logging_file_handler)

    def save_summary_info(
        self,
        episode_info: list[StepInfo],
        exp_dir: Path,
        err_msg: str | None,
        stack_trace: str | None,
    ):
        # bring err from agent_info to the top level
        if err_msg is None:
            err_msg, stack_trace = _extract_err_msg(episode_info)
        else:
            # useful until we get a proper place in agent_xray to view error
            # messages.
            if len(episode_info) == 0:
                episode_info.append(StepInfo())


        stream_handler = logging.StreamHandler()
        stream_handler.setLevel(self.logging_level_stdout)
        stream_handler.setFormatter(formatter)
        # setup root logger
        root_logger = logging.getLogger()

        # remove previous stream handlers
        for handler in root_logger.handlers:
            if isinstance(handler, logging.StreamHandler):
                root_logger.removeHandler(handler)

        root_logger.setLevel(self.logging_level)
        root_logger.addHandler(file_handler)
        root_logger.addHandler(stream_handler)
        # setup openai logger (don't go below INFO verbosity)
        openai_logger = logging.getLogger("openai._base_client")
        openai_logger.setLevel(max(logging.INFO, self.logging_level))

        self.logging_file_handler = file_handler

    def _unset_logger(self):
        root_logger = logging.getLogger()
        root_logger.removeHandler(self.logging_file_handler)

    def save_summary_info(
        self,
        episode_info: list[StepInfo],
        exp_dir: Path,
        err_msg: str | None,
        stack_trace: str | None,
    ):
        # bring err from agent_info to the top level
        if err_msg is None:
            err_msg, stack_trace = _extract_err_msg(episode_info)
        else:
            # useful until we get a proper place in agent_xray to view error
            # messages.
            if len(episode_info) == 0:
                episode_info.append(StepInfo())
            episode_info[-1].agent_info["err_msg"] = err_msg
            episode_info[-1].agent_info["stack_trace"] = stack_trace

        summary_info = dict(


        root_logger = logging.getLogger()

        # remove previous stream handlers
        for handler in root_logger.handlers:
            if isinstance(handler, logging.StreamHandler):
                root_logger.removeHandler(handler)

        root_logger.setLevel(self.logging_level)
        root_logger.addHandler(file_handler)
        root_logger.addHandler(stream_handler)
        # setup openai logger (don't go below INFO verbosity)
        openai_logger = logging.getLogger("openai._base_client")
        openai_logger.setLevel(max(logging.INFO, self.logging_level))

        self.logging_file_handler = file_handler

    def _unset_logger(self):
        root_logger = logging.getLogger()
        root_logger.removeHandler(self.logging_file_handler)

    def save_summary_info(
        self,
        episode_info: list[StepInfo],
        exp_dir: Path,
        err_msg: str | None,
        stack_trace: str | None,
    ):
        # bring err from agent_info to the top level
        if err_msg is None:
            err_msg, stack_trace = _extract_err_msg(episode_info)
        else:
            # useful until we get a proper place in agent_xray to view error
            # messages.
            if len(episode_info) == 0:
                episode_info.append(StepInfo())
            episode_info[-1].agent_info["err_msg"] = err_msg
            episode_info[-1].agent_info["stack_trace"] = stack_trace

        summary_info = dict(
            n_steps=len(episode_info) - 1,
            cum_reward=sum([step.reward for step in episode_info]),
            cum_raw_reward=sum([step.raw_reward for step in episode_info if step.raw_reward]),
            err_msg=err_msg,
            stack_trace=stack_trace,
        )
        for key, val in _aggregate_episode_stats(episode_info).items():
            summary_info[f"stats.{key}"] = val

        if len(episode_info) > 0:
            summary_info["terminated"] = episode_info[-1].terminated
            summary_info["truncated"] = episode_info[-1].truncated

        with open(exp_dir / "summary_info.json", "w") as f:
            json.dump(summary_info, f, indent=4)

def _extract_err_msg(episode_info: list[StepInfo]):
    """Extract the last error message from the episode info."""
    errors = [(None, None)]
    for step_info in episode_info:
        if step_info.agent_info is None:
            continue
        err_msg = step_info.agent_info.get("err_msg", None)
        if err_msg is not None:
            errors.append((err_msg, step_info.agent_info.get("stack_trace", None)))

    return errors[-1]

def _aggregate_episode_stats(episode_info: list[StepInfo]):
    """Aggregate StepInfo.stats across episodes.

    It will compute the sum and max of each value in the stats dict.
    These two summaries should cover many use cases. If more are needed, the


class ExpResult:
    """Helper class to load and visualize the results of an experiment.

    attributes are loaded lazily.

    Attributes (lazily loaded):
        exp_args: ExpArgs, the arguments of the experiment.
        steps_info: list[StepInfo], the information of each steps so far
        summary_info: dict, the summary of the experiment.
        screenshots: list[Image], the screenshots of each step.
        screenshots_som: list[Image], the screenshots of each step with set of
            marks inprinted.
        flat_exp_args: dict, the flattened version of exp_args.
        chat_video_path: Path, the path to the chat video. (if record_video=True)
        task_video_path: Path, the path to the task video. (if record_video=True)
        combined_video_path: Path, the path to the combined video. (if video was
            combined)
    """

    def __init__(self, exp_dir) -> None:
        self.exp_dir = Path(exp_dir)
        self._exp_args = None
        self._steps_info = {}
        self._summary_info = None
        self._screenshots = {}
        self._flat_exp_args = None
        self._logs = None

    @property
    def exp_args(self) -> ExpArgs:
        if self._exp_args is None:
            with open(self.exp_dir / "exp_args.pkl", "rb") as f:
                self._exp_args = pickle.load(f)
                # in case experiments were moved
                self._exp_args.exp_dir = self.exp_dir
        return self._exp_args

    def get_step_info(self, step: int) -> StepInfo:
        """Load the step info from the file and return it."""
        if self._steps_info.get(step, None) is None:
            with gzip.open(self.exp_dir / f"step_{step}.pkl.gz", "rb") as f:
                self._steps_info[step] = pickle.load(f)
            if self._steps_info[step].obs:
                if "screenshot" not in self._steps_info[step].obs:
                    try:
                        self._steps_info[step].obs["screenshot"] = np.array(
                            self.get_screenshot(step), dtype=np.uint8


        screenshots: list[Image], the screenshots of each step.
        screenshots_som: list[Image], the screenshots of each step with set of
            marks inprinted.
        flat_exp_args: dict, the flattened version of exp_args.
        chat_video_path: Path, the path to the chat video. (if record_video=True)
        task_video_path: Path, the path to the task video. (if record_video=True)
        combined_video_path: Path, the path to the combined video. (if video was
            combined)
    """

    def __init__(self, exp_dir) -> None:
        self.exp_dir = Path(exp_dir)
        self._exp_args = None
        self._steps_info = {}
        self._summary_info = None
        self._screenshots = {}
        self._flat_exp_args = None
        self._logs = None

    @property
    def exp_args(self) -> ExpArgs:
        if self._exp_args is None:
            with open(self.exp_dir / "exp_args.pkl", "rb") as f:
                self._exp_args = pickle.load(f)
                # in case experiments were moved
                self._exp_args.exp_dir = self.exp_dir
        return self._exp_args

    def get_step_info(self, step: int) -> StepInfo:
        """Load the step info from the file and return it."""
        if self._steps_info.get(step, None) is None:
            with gzip.open(self.exp_dir / f"step_{step}.pkl.gz", "rb") as f:
                self._steps_info[step] = pickle.load(f)
            if self._steps_info[step].obs:
                if "screenshot" not in self._steps_info[step].obs:
                    try:
                        self._steps_info[step].obs["screenshot"] = np.array(
                            self.get_screenshot(step), dtype=np.uint8
                        )
                    except FileNotFoundError:
                        pass
                if "screenshot_som" not in self._steps_info[step].obs:
                    try:
                        self._steps_info[step].obs["screenshot_som"] = np.array(
                            self.get_screenshot(step, som=True), dtype=np.uint8
                        )
                    except FileNotFoundError:


    """

    def __init__(self, exp_dir) -> None:
        self.exp_dir = Path(exp_dir)
        self._exp_args = None
        self._steps_info = {}
        self._summary_info = None
        self._screenshots = {}
        self._flat_exp_args = None
        self._logs = None

    @property
    def exp_args(self) -> ExpArgs:
        if self._exp_args is None:
            with open(self.exp_dir / "exp_args.pkl", "rb") as f:
                self._exp_args = pickle.load(f)
                # in case experiments were moved
                self._exp_args.exp_dir = self.exp_dir
        return self._exp_args

    def get_step_info(self, step: int) -> StepInfo:
        """Load the step info from the file and return it."""
        if self._steps_info.get(step, None) is None:
            with gzip.open(self.exp_dir / f"step_{step}.pkl.gz", "rb") as f:
                self._steps_info[step] = pickle.load(f)
            if self._steps_info[step].obs:
                if "screenshot" not in self._steps_info[step].obs:
                    try:
                        self._steps_info[step].obs["screenshot"] = np.array(
                            self.get_screenshot(step), dtype=np.uint8
                        )
                    except FileNotFoundError:
                        pass
                if "screenshot_som" not in self._steps_info[step].obs:
                    try:
                        self._steps_info[step].obs["screenshot_som"] = np.array(
                            self.get_screenshot(step, som=True), dtype=np.uint8
                        )
                    except FileNotFoundError:
                        pass
                # if goal_object is set to None, it indicates it has been saved into a separate file
                if (
                    "goal_object" in self._steps_info[step].obs
                    and self._steps_info[step].obs["goal_object"] is None
                ):
                    with gzip.open(self.exp_dir / "goal_object.pkl.gz", "rb") as f:
                        goal_object = pickle.load(f)
                        self._steps_info[step].obs["goal_object"] = goal_object

        return self._steps_info[step]

    @property
    def steps_info(self) -> list[StepInfo]:
        step_files = list(self.exp_dir.glob("step_*.pkl.gz"))
        for file in step_files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_step_info(step)

        return [self._steps_info[i] for i in range(len(self._steps_info))]

    @property
    def summary_info(self) -> dict:
        if self._summary_info is None:
            with open(self.exp_dir / "summary_info.json", "r") as f:
                # if length is zero raise file not found error
                if os.fstat(f.fileno()).st_size == 0:
                    raise FileNotFoundError("summary_info.json is empty.")
                self._summary_info = json.load(f)
        return self._summary_info


                        pass
                if "screenshot_som" not in self._steps_info[step].obs:
                    try:
                        self._steps_info[step].obs["screenshot_som"] = np.array(
                            self.get_screenshot(step, som=True), dtype=np.uint8
                        )
                    except FileNotFoundError:
                        pass
                # if goal_object is set to None, it indicates it has been saved into a separate file
                if (
                    "goal_object" in self._steps_info[step].obs
                    and self._steps_info[step].obs["goal_object"] is None
                ):
                    with gzip.open(self.exp_dir / "goal_object.pkl.gz", "rb") as f:
                        goal_object = pickle.load(f)
                        self._steps_info[step].obs["goal_object"] = goal_object

        return self._steps_info[step]

    @property
    def steps_info(self) -> list[StepInfo]:
        step_files = list(self.exp_dir.glob("step_*.pkl.gz"))
        for file in step_files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_step_info(step)

        return [self._steps_info[i] for i in range(len(self._steps_info))]

    @property
    def summary_info(self) -> dict:
        if self._summary_info is None:
            with open(self.exp_dir / "summary_info.json", "r") as f:
                # if length is zero raise file not found error
                if os.fstat(f.fileno()).st_size == 0:
                    raise FileNotFoundError("summary_info.json is empty.")
                self._summary_info = json.load(f)
        return self._summary_info

    def get_screenshot(self, step: int, som=False) -> Image:
        key = (step, som)
        if self._screenshots.get(key, None) is None:
            file_path = self.get_screenshot_path(step, som=som)
            self._screenshots[key] = Image.open(file_path).convert("RGB")
        return self._screenshots[key]

    def get_screenshot_path(self, step: int, som=False) -> Path:
        """Return the path to the screenshot file."""


                if (
                    "goal_object" in self._steps_info[step].obs
                    and self._steps_info[step].obs["goal_object"] is None
                ):
                    with gzip.open(self.exp_dir / "goal_object.pkl.gz", "rb") as f:
                        goal_object = pickle.load(f)
                        self._steps_info[step].obs["goal_object"] = goal_object

        return self._steps_info[step]

    @property
    def steps_info(self) -> list[StepInfo]:
        step_files = list(self.exp_dir.glob("step_*.pkl.gz"))
        for file in step_files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_step_info(step)

        return [self._steps_info[i] for i in range(len(self._steps_info))]

    @property
    def summary_info(self) -> dict:
        if self._summary_info is None:
            with open(self.exp_dir / "summary_info.json", "r") as f:
                # if length is zero raise file not found error
                if os.fstat(f.fileno()).st_size == 0:
                    raise FileNotFoundError("summary_info.json is empty.")
                self._summary_info = json.load(f)
        return self._summary_info

    def get_screenshot(self, step: int, som=False) -> Image:
        key = (step, som)
        if self._screenshots.get(key, None) is None:
            file_path = self.get_screenshot_path(step, som=som)
            self._screenshots[key] = Image.open(file_path).convert("RGB")
        return self._screenshots[key]

    def get_screenshot_path(self, step: int, som=False) -> Path:
        """Return the path to the screenshot file."""
        file_name = f"screenshot_{'som_' if som else ''}step_{step}"
        for ext in [".png", ".jpg"]:
            file_path = self.exp_dir / (file_name + ext)
            if file_path.exists():
                return file_path
        raise FileNotFoundError(
            f"No screenshot found for step {step} (som={som}) in {self.exp_dir}"
        )

    def get_screenshots(self, som=False):


    @property
    def steps_info(self) -> list[StepInfo]:
        step_files = list(self.exp_dir.glob("step_*.pkl.gz"))
        for file in step_files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_step_info(step)

        return [self._steps_info[i] for i in range(len(self._steps_info))]

    @property
    def summary_info(self) -> dict:
        if self._summary_info is None:
            with open(self.exp_dir / "summary_info.json", "r") as f:
                # if length is zero raise file not found error
                if os.fstat(f.fileno()).st_size == 0:
                    raise FileNotFoundError("summary_info.json is empty.")
                self._summary_info = json.load(f)
        return self._summary_info

    def get_screenshot(self, step: int, som=False) -> Image:
        key = (step, som)
        if self._screenshots.get(key, None) is None:
            file_path = self.get_screenshot_path(step, som=som)
            self._screenshots[key] = Image.open(file_path).convert("RGB")
        return self._screenshots[key]

    def get_screenshot_path(self, step: int, som=False) -> Path:
        """Return the path to the screenshot file."""
        file_name = f"screenshot_{'som_' if som else ''}step_{step}"
        for ext in [".png", ".jpg"]:
            file_path = self.exp_dir / (file_name + ext)
            if file_path.exists():
                return file_path
        raise FileNotFoundError(
            f"No screenshot found for step {step} (som={som}) in {self.exp_dir}"
        )

    def get_screenshots(self, som=False):
        files = list(self.exp_dir.glob("screenshot_step_*"))
        max_step = 0
        for file in files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_screenshot(step, som=som)
            max_step = max(max_step, step)
        return [self._screenshots.get((i, som), None) for i in range(max_step + 1)]


        return [self._steps_info[i] for i in range(len(self._steps_info))]

    @property
    def summary_info(self) -> dict:
        if self._summary_info is None:
            with open(self.exp_dir / "summary_info.json", "r") as f:
                # if length is zero raise file not found error
                if os.fstat(f.fileno()).st_size == 0:
                    raise FileNotFoundError("summary_info.json is empty.")
                self._summary_info = json.load(f)
        return self._summary_info

    def get_screenshot(self, step: int, som=False) -> Image:
        key = (step, som)
        if self._screenshots.get(key, None) is None:
            file_path = self.get_screenshot_path(step, som=som)
            self._screenshots[key] = Image.open(file_path).convert("RGB")
        return self._screenshots[key]

    def get_screenshot_path(self, step: int, som=False) -> Path:
        """Return the path to the screenshot file."""
        file_name = f"screenshot_{'som_' if som else ''}step_{step}"
        for ext in [".png", ".jpg"]:
            file_path = self.exp_dir / (file_name + ext)
            if file_path.exists():
                return file_path
        raise FileNotFoundError(
            f"No screenshot found for step {step} (som={som}) in {self.exp_dir}"
        )

    def get_screenshots(self, som=False):
        files = list(self.exp_dir.glob("screenshot_step_*"))
        max_step = 0
        for file in files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_screenshot(step, som=som)
            max_step = max(max_step, step)
        return [self._screenshots.get((i, som), None) for i in range(max_step + 1)]

    @property
    def screenshots(self):
        return self.get_screenshots(som=False)

    @property
    def screenshots_som(self):
        return self.get_screenshots(som=True)

    @property
    def flat_exp_args(self) -> dict:


        return self._summary_info

    def get_screenshot(self, step: int, som=False) -> Image:
        key = (step, som)
        if self._screenshots.get(key, None) is None:
            file_path = self.get_screenshot_path(step, som=som)
            self._screenshots[key] = Image.open(file_path).convert("RGB")
        return self._screenshots[key]

    def get_screenshot_path(self, step: int, som=False) -> Path:
        """Return the path to the screenshot file."""
        file_name = f"screenshot_{'som_' if som else ''}step_{step}"
        for ext in [".png", ".jpg"]:
            file_path = self.exp_dir / (file_name + ext)
            if file_path.exists():
                return file_path
        raise FileNotFoundError(
            f"No screenshot found for step {step} (som={som}) in {self.exp_dir}"
        )

    def get_screenshots(self, som=False):
        files = list(self.exp_dir.glob("screenshot_step_*"))
        max_step = 0
        for file in files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_screenshot(step, som=som)
            max_step = max(max_step, step)
        return [self._screenshots.get((i, som), None) for i in range(max_step + 1)]

    @property
    def screenshots(self):
        return self.get_screenshots(som=False)

    @property
    def screenshots_som(self):
        return self.get_screenshots(som=True)

    @property
    def flat_exp_args(self) -> dict:
        """Return a dict with exp_args flattened."""
        if self._flat_exp_args is None:
            exp_args = asdict(self.exp_args)
            # this will flatten nested dicts
            self._flat_exp_args = _flatten_dict(exp_args)
        return self._flat_exp_args

    def get_exp_record(self) -> dict:
        """Return a dict with exp_args flattened and summary_info."""


screenshots

        """Return the path to the screenshot file."""
        file_name = f"screenshot_{'som_' if som else ''}step_{step}"
        for ext in [".png", ".jpg"]:
            file_path = self.exp_dir / (file_name + ext)
            if file_path.exists():
                return file_path
        raise FileNotFoundError(
            f"No screenshot found for step {step} (som={som}) in {self.exp_dir}"
        )

    def get_screenshots(self, som=False):
        files = list(self.exp_dir.glob("screenshot_step_*"))
        max_step = 0
        for file in files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_screenshot(step, som=som)
            max_step = max(max_step, step)
        return [self._screenshots.get((i, som), None) for i in range(max_step + 1)]

    @property
    def screenshots(self):
        return self.get_screenshots(som=False)

    @property
    def screenshots_som(self):
        return self.get_screenshots(som=True)

    @property
    def flat_exp_args(self) -> dict:
        """Return a dict with exp_args flattened."""
        if self._flat_exp_args is None:
            exp_args = asdict(self.exp_args)
            # this will flatten nested dicts
            self._flat_exp_args = _flatten_dict(exp_args)
        return self._flat_exp_args

    def get_exp_record(self) -> dict:
        """Return a dict with exp_args flattened and summary_info."""
        record = {"exp_dir": self.exp_dir}
        try:
            record.update(self.flat_exp_args)
        except FileNotFoundError:


            if file_path.exists():
                return file_path
        raise FileNotFoundError(
            f"No screenshot found for step {step} (som={som}) in {self.exp_dir}"
        )

    def get_screenshots(self, som=False):
        files = list(self.exp_dir.glob("screenshot_step_*"))
        max_step = 0
        for file in files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_screenshot(step, som=som)
            max_step = max(max_step, step)
        return [self._screenshots.get((i, som), None) for i in range(max_step + 1)]

    @property
    def screenshots(self):
        return self.get_screenshots(som=False)

    @property
    def screenshots_som(self):
        return self.get_screenshots(som=True)

    @property
    def flat_exp_args(self) -> dict:
        """Return a dict with exp_args flattened."""
        if self._flat_exp_args is None:
            exp_args = asdict(self.exp_args)
            # this will flatten nested dicts
            self._flat_exp_args = _flatten_dict(exp_args)
        return self._flat_exp_args

    def get_exp_record(self) -> dict:
        """Return a dict with exp_args flattened and summary_info."""
        record = {"exp_dir": self.exp_dir}
        try:
            record.update(self.flat_exp_args)
        except FileNotFoundError:
            pass
        try:
            record.update(self.summary_info)
        except FileNotFoundError:


        )

    def get_screenshots(self, som=False):
        files = list(self.exp_dir.glob("screenshot_step_*"))
        max_step = 0
        for file in files:
            step = int(file.name.split("_")[-1].split(".")[0])
            self.get_screenshot(step, som=som)
            max_step = max(max_step, step)
        return [self._screenshots.get((i, som), None) for i in range(max_step + 1)]

    @property
    def screenshots(self):
        return self.get_screenshots(som=False)

    @property
    def screenshots_som(self):
        return self.get_screenshots(som=True)

    @property
    def flat_exp_args(self) -> dict:
        """Return a dict with exp_args flattened."""
        if self._flat_exp_args is None:
            exp_args = asdict(self.exp_args)
            # this will flatten nested dicts
            self._flat_exp_args = _flatten_dict(exp_args)
        return self._flat_exp_args

    def get_exp_record(self) -> dict:
        """Return a dict with exp_args flattened and summary_info."""
        record = {"exp_dir": self.exp_dir}
        try:
            record.update(self.flat_exp_args)
        except FileNotFoundError:
            pass
        try:
            record.update(self.summary_info)
        except FileNotFoundError:
            pass
        return record

    @property
    def chat_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("chat_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No chat_video found in {self.exp_dir}")


            max_step = max(max_step, step)
        return [self._screenshots.get((i, som), None) for i in range(max_step + 1)]

    @property
    def screenshots(self):
        return self.get_screenshots(som=False)

    @property
    def screenshots_som(self):
        return self.get_screenshots(som=True)

    @property
    def flat_exp_args(self) -> dict:
        """Return a dict with exp_args flattened."""
        if self._flat_exp_args is None:
            exp_args = asdict(self.exp_args)
            # this will flatten nested dicts
            self._flat_exp_args = _flatten_dict(exp_args)
        return self._flat_exp_args

    def get_exp_record(self) -> dict:
        """Return a dict with exp_args flattened and summary_info."""
        record = {"exp_dir": self.exp_dir}
        try:
            record.update(self.flat_exp_args)
        except FileNotFoundError:
            pass
        try:
            record.update(self.summary_info)
        except FileNotFoundError:
            pass
        return record

    @property
    def chat_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("chat_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No chat_video found in {self.exp_dir}")

    @property
    def task_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("task_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No task_video found in {self.exp_dir}")

    @property
    def combined_video_path(self) -> Path:
        return self.exp_dir / "combined_video.mp4"

    @property


        if self._flat_exp_args is None:
            exp_args = asdict(self.exp_args)
            # this will flatten nested dicts
            self._flat_exp_args = _flatten_dict(exp_args)
        return self._flat_exp_args

    def get_exp_record(self) -> dict:
        """Return a dict with exp_args flattened and summary_info."""
        record = {"exp_dir": self.exp_dir}
        try:
            record.update(self.flat_exp_args)
        except FileNotFoundError:
            pass
        try:
            record.update(self.summary_info)
        except FileNotFoundError:
            pass
        return record

    @property
    def chat_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("chat_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No chat_video found in {self.exp_dir}")

    @property
    def task_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("task_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No task_video found in {self.exp_dir}")

    @property
    def combined_video_path(self) -> Path:
        return self.exp_dir / "combined_video.mp4"

    @property
    def logs(self):
        if self._logs is None:
            self._logs = (self.exp_dir / "experiment.log").read_text()
        return self._logs

    @property
    def status(self):


        """Return a dict with exp_args flattened and summary_info."""
        record = {"exp_dir": self.exp_dir}
        try:
            record.update(self.flat_exp_args)
        except FileNotFoundError:
            pass
        try:
            record.update(self.summary_info)
        except FileNotFoundError:
            pass
        return record

    @property
    def chat_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("chat_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No chat_video found in {self.exp_dir}")

    @property
    def task_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("task_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No task_video found in {self.exp_dir}")

    @property
    def combined_video_path(self) -> Path:
        return self.exp_dir / "combined_video.mp4"

    @property
    def logs(self):
        if self._logs is None:
            self._logs = (self.exp_dir / "experiment.log").read_text()
        return self._logs

    @property
    def status(self):
        """Possible values:
        * "done": completed with no error
        * "error": completed with error
        * "incomplete": not completed yet (may be pending or just stalled)

        Returns:
            str: the status of the experiment. One of "done", "error", "incomplete".


            record.update(self.summary_info)
        except FileNotFoundError:
            pass
        return record

    @property
    def chat_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("chat_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No chat_video found in {self.exp_dir}")

    @property
    def task_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("task_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No task_video found in {self.exp_dir}")

    @property
    def combined_video_path(self) -> Path:
        return self.exp_dir / "combined_video.mp4"

    @property
    def logs(self):
        if self._logs is None:
            self._logs = (self.exp_dir / "experiment.log").read_text()
        return self._logs

    @property
    def status(self):
        """Possible values:
        * "done": completed with no error
        * "error": completed with error
        * "incomplete": not completed yet (may be pending or just stalled)

        Returns:
            str: the status of the experiment. One of "done", "error", "incomplete".
        """
        try:
            summary_info = self.summary_info
        except FileNotFoundError:


logs

    @property
    def chat_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("chat_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No chat_video found in {self.exp_dir}")

    @property
    def task_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("task_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No task_video found in {self.exp_dir}")

    @property
    def combined_video_path(self) -> Path:
        return self.exp_dir / "combined_video.mp4"

    @property
    def logs(self):
        if self._logs is None:
            self._logs = (self.exp_dir / "experiment.log").read_text()
        return self._logs

    @property
    def status(self):
        """Possible values:
        * "done": completed with no error
        * "error": completed with error
        * "incomplete": not completed yet (may be pending or just stalled)

        Returns:
            str: the status of the experiment. One of "done", "error", "incomplete".
        """
        try:
            summary_info = self.summary_info
        except FileNotFoundError:
            return "incomplete"

        if summary_info.get("err_msg", None) is not None:
            return "error"

        if summary_info.get("terminated", False) or summary_info.get("truncated", False):


status

            raise FileNotFoundError(f"No chat_video found in {self.exp_dir}")

    @property
    def task_video_path(self) -> Path:
        try:
            return next(self.exp_dir.glob("task_video/*.webm"))
        except StopIteration:
            raise FileNotFoundError(f"No task_video found in {self.exp_dir}")

    @property
    def combined_video_path(self) -> Path:
        return self.exp_dir / "combined_video.mp4"

    @property
    def logs(self):
        if self._logs is None:
            self._logs = (self.exp_dir / "experiment.log").read_text()
        return self._logs

    @property
    def status(self):
        """Possible values:
        * "done": completed with no error
        * "error": completed with error
        * "incomplete": not completed yet (may be pending or just stalled)

        Returns:
            str: the status of the experiment. One of "done", "error", "incomplete".
        """
        try:
            summary_info = self.summary_info
        except FileNotFoundError:
            return "incomplete"

        if summary_info.get("err_msg", None) is not None:
            return "error"

        if summary_info.get("terminated", False) or summary_info.get("truncated", False):
            return "done"

        return "incomplete"

EXP_RESULT_CACHE = {}

def get_exp_result(exp_dir) -> ExpResult:
    """Keep a cache of pre-loaded exp_results for faster loading"""
    exp_dir = str(exp_dir)  # make sure it's not a Path
    exp_result = EXP_RESULT_CACHE.get(exp_dir, None)
    if exp_result is None:
        exp_result = ExpResult(exp_dir)
        EXP_RESULT_CACHE[exp_dir] = exp_result
    return exp_result

def yield_all_exp_results(
    savedir_base: str | Path, progress_fn=tqdm, load_hidden=False, use_cache=True
):
    """Recursively find all experiments from savedir_base folder.


default

    exp_args_paths = []
    for exp_dir in savedir_base:
        exp_args_paths.extend(list(Path(exp_dir).glob("**/exp_args.pkl")))

    if progress_fn is not None:
        exp_args_paths = progress_fn(exp_args_paths, desc="Searching experiments directories.")

    for exp_args_path in exp_args_paths:
        exp_dir = exp_args_path.parent
        if not load_hidden:
            if exp_dir.name.startswith("_") or exp_dir.name.startswith("."):
                continue
        if use_cache:
            yield get_exp_result(exp_dir)
        else:
            yield ExpResult(exp_dir)

class DataclassJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if is_dataclass(obj):
            return asdict(obj)
        if isinstance(obj, np.integer):
            return int(obj)
        if isinstance(obj, np.floating):
            return float(obj)
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        return super().default(obj)

def _move_old_exp(exp_dir):
    """Move the old experiment directory to a new name."""
    exp_dir = Path(exp_dir)
    if exp_dir.exists():
        exp_dir.rename(exp_dir.with_name("_" + exp_dir.name))

def _get_env_name(task_name: str):
    """Register tasks if needed (lazy import) and return environment name."""

    # lazy import
    if task_name.startswith("miniwob"):
        import browsergym.miniwob
    elif task_name.startswith("workarena"):
        import browsergym.workarena
    elif task_name.startswith("webarena"):
        import browsergym.webarena
    elif task_name.startswith("visualwebarena"):


src.agentlab.experiments.args

import copy
from abc import ABC
from dataclasses import fields, is_dataclass
from itertools import product
from typing import Any

import numpy as np

class CrossProd:
    """Use to specify that this will be part of a cross product"""

    def __init__(self, elements):
        self.elements = elements

class Distribution(ABC):
    """Generic Class to identify that this is a distribution"""

    def sample(self):
        pass

class Choice(Distribution):
    """Use to specify that this will be sampled from a list of elements"""

    def __init__(self, elements, p=None):
        self.elements = elements
        self.p = p

    def sample(self, rng=np.random):
        return rng.choice(self.elements, p=self.p)

def _find_cprod_with_paths(obj, path=None):
    """Find all the CrossProd objects and their paths in the given object.

    Args:
        obj (Any): The object to search for CrossProd objects.
        path (List[str]): The path to the current object.

    Returns:
        List[Tuple[List[str], CrossProd]]: A list of tuples where the first element is the path to the CrossProd
            object and the second element is the CrossProd object.
    """
    if path is None:
        path = []

    if isinstance(obj, CrossProd):
        return [(path, obj)]

    cprod_paths = []
    if is_dataclass(obj):
        for field in fields(obj):
            field_value = getattr(obj, field.name)
            cprod_paths += _find_cprod_with_paths(field_value, path + [field.name])
    elif isinstance(obj, dict):
        for key, value in obj.items():
            cprod_paths += _find_cprod_with_paths(value, path + [key])

    return cprod_paths

def _set_value(obj, path, value):
    """Set the value of the given path in the given object to the given value."""
    for key in path[:-1]:
        if isinstance(obj, dict):
            obj = obj[key]
        else:
            obj = getattr(obj, key)
    if isinstance(obj, dict):
        obj[path[-1]] = value
    else:
        setattr(obj, path[-1], value)

def expand_cross_product(obj: Any | list[Any]):
    """Expand the given object into a list of objects with all combinations of
    CrossProd objects.

    This function will recursively search for CrossProd objects in the given
    object and create a list of objects with all combinations of CrossProd. It
    searches through dataclasses and dictionaries.

    Args:
        obj: Any | List[Any],
            The object to expand.

    Returns:
        List[Any]:
        A list of objects with all combinations of CrossProd objects.

    """

    if isinstance(obj, CrossProd):
        return obj.elements

    if isinstance(obj, list):
        obj_list = obj
    else:
        obj_list = [obj]

    result = []

    for obj in obj_list:
        cprod_paths = _find_cprod_with_paths(obj)
        if not cprod_paths:
            result.append(copy.deepcopy(obj))
            continue

        paths, cprod_objects = zip(*cprod_paths)
        combinations = product(*[cprod_obj.elements for cprod_obj in cprod_objects])

        # create a base object with empty fields to make fast deep copies from
        base_obj = copy.deepcopy(obj)
        for path in paths:
            _set_value(base_obj, path, None)

        for combo in combinations:
            new_obj = copy.deepcopy(base_obj)
            for path, value in zip(paths, combo):
                _set_value(new_obj, path, value)
            result.append(new_obj)

    return result

def sample_and_expand_cross_product(obj: Any | list[Any], n_samples: int):
    """This will sample first and then expand the cross product."""
    return expand_cross_product(sample_args(obj, n_samples))

def sample_args(obj: Any | list[Any], n_samples: int):
    """Sample the given object n_samples times. Each sample is a deep copy of
    the original object with any object of type Distribution replaced by a
    sample from that distribution.

    Args:
        obj: Any | List[Any],
            the object to sample
        n_samples: int,
            the number of samples to generate

    Returns:
        List[Any]:
        A list of n_samples objects with the given object.
    """

    if isinstance(obj, list):
        obj_list = obj
    else:
        obj_list = [obj]

    result = []

    for obj in obj_list:
        for _ in range(n_samples):
            result.append(_sample_single(copy.deepcopy(obj)))

    return result

def _sample_single(obj):
    """Sample the given object once."""
    if isinstance(obj, Distribution):
        return obj.sample()

    if is_dataclass(obj):
        for field in fields(obj):
            value = getattr(obj, field.name)
            sampled_value = _sample_single(value)
            setattr(obj, field.name, sampled_value)
    elif isinstance(obj, dict):
        for key, value in obj.items():
            obj[key] = _sample_single(value)

    return obj

class Toggle:
    pass

TOGGLE = Toggle()

def _change_value(obj, path, value):
    """Set the value to the given path in the nested objects.

    Note: This doesn't work with list or dict or tuples. Only works with objects.

    Args:
        obj: the object to change the value
        path: the path to the value to change
        value: the value to change to

    Raises:
        ValueError: if the field is not found in the object
    """
    key_list = path.split(".")
    for key in key_list[:-1]:
        if key == "":
            continue
        obj = getattr(obj, key)
    # if dataclass, then set the value only if the field exists

    def _set(obj, key, value):
        if isinstance(value, Toggle):
            previous_value = getattr(obj, key)
            if not isinstance(previous_value, bool):
                raise ValueError(f"Toggle object {obj} attribute {key} is not a boolean")
            setattr(obj, key, not previous_value)
        else:
            setattr(obj, key_list[-1], value)

    if is_dataclass(obj):
        field_names = [field.name for field in fields(obj)]
        if key_list[-1] in field_names:
            _set(obj, key_list[-1], value)
        else:
            raise ValueError(f"field {key_list[-1]} not found in {obj}")
    else:
        _set(obj, key_list[-1], value)

def _apply_change(params, change):
    """Apply the change to the params object in place."""
    if callable(change):
        change(params)
    elif isinstance(change, (tuple, list)):
        if isinstance(change[0], str) and len(change) == 2:
            path, value = change
            _change_value(params, path, value)
        else:
            for c in change:
                _apply_change(params, c)
    else:
        raise ValueError(f"change {change} not recognized")
    return params

def make_progression_study(start_point, changes, return_cross_prod=True):
    """A kind of ablation study by changing the start_point with changes one by
    one.

    Args:
        start_point: the starting point of the study
        changes: a list of changes to make to the start_point. Each change is
            either a callable or tuple containing a string identifying the path in the object to
            change and the value to change to. ex: (".obs.use_html", True)
        return_cross_prod: return a CrossProd object or just the list

    Returns:
        A CrossProd object containing a list of objects with progressive changes
        from `start_point`. If return_cross_prod is False, then it will return a
        list.

    """
    params_list = [start_point]
    for change in changes:
        params = copy.deepcopy(params_list[-1])
        _apply_change(params, change)
        params_list.append(params)

    if return_cross_prod:
        return CrossProd(params_list)
    else:
        return params_list

def make_ablation_study(start_point, changes, return_cross_prod=True):
    """Ablation study by modifying the start_point with only one change at a
    time, and restarting from the original start_point for each configuration.

    Args:
        start_point: the starting point of the ablation study
        changes: a list of changes to make to the start_point. Each change is
            either a callable or tuple containing a string identifying the path in the object to
            change and the value to change to. ex: (".obs.use_html", True)
        return_cross_prod: return a CrossProd object or just the list

    Returns:
        A CrossProd object containing a list of objects with one change
        from `start_point`. If return_cross_prod is False, then it will return a
        list.

    """
    params_list = [start_point]
    for change in changes:
        params = copy.deepcopy(start_point)
        _apply_change(params, change)
        params_list.append(params)

    if return_cross_prod:
        return CrossProd(params_list)
    else:
        return params_list

if __name__ == "__main__":
    from agentlab.agents.dynamic_prompting import Flags

    study = make_progression_study(
        start_point=Flags(),
        changes=[
            ("use_thinking", True),
            (".use_plan", True),
            (".use_criticise", True),
        ],
        return_cross_prod=True,
    )

    study = expand_cross_product(study)
    for p in study:
        print(p)


CrossProd

import copy
from abc import ABC
from dataclasses import fields, is_dataclass
from itertools import product
from typing import Any

import numpy as np

class CrossProd:
    """Use to specify that this will be part of a cross product"""

    def __init__(self, elements):
        self.elements = elements

class Distribution(ABC):
    """Generic Class to identify that this is a distribution"""

    def sample(self):
        pass

class Choice(Distribution):
    """Use to specify that this will be sampled from a list of elements"""

    def __init__(self, elements, p=None):
        self.elements = elements
        self.p = p

    def sample(self, rng=np.random):
        return rng.choice(self.elements, p=self.p)


Distribution

import copy
from abc import ABC
from dataclasses import fields, is_dataclass
from itertools import product
from typing import Any

import numpy as np

class CrossProd:
    """Use to specify that this will be part of a cross product"""

    def __init__(self, elements):
        self.elements = elements

class Distribution(ABC):
    """Generic Class to identify that this is a distribution"""

    def sample(self):
        pass

class Choice(Distribution):
    """Use to specify that this will be sampled from a list of elements"""

    def __init__(self, elements, p=None):
        self.elements = elements
        self.p = p

    def sample(self, rng=np.random):
        return rng.choice(self.elements, p=self.p)

def _find_cprod_with_paths(obj, path=None):
    """Find all the CrossProd objects and their paths in the given object.

    Args:
        obj (Any): The object to search for CrossProd objects.
        path (List[str]): The path to the current object.


Choice

from itertools import product
from typing import Any

import numpy as np

class CrossProd:
    """Use to specify that this will be part of a cross product"""

    def __init__(self, elements):
        self.elements = elements

class Distribution(ABC):
    """Generic Class to identify that this is a distribution"""

    def sample(self):
        pass

class Choice(Distribution):
    """Use to specify that this will be sampled from a list of elements"""

    def __init__(self, elements, p=None):
        self.elements = elements
        self.p = p

    def sample(self, rng=np.random):
        return rng.choice(self.elements, p=self.p)

def _find_cprod_with_paths(obj, path=None):
    """Find all the CrossProd objects and their paths in the given object.

    Args:
        obj (Any): The object to search for CrossProd objects.
        path (List[str]): The path to the current object.

    Returns:
        List[Tuple[List[str], CrossProd]]: A list of tuples where the first element is the path to the CrossProd
            object and the second element is the CrossProd object.
    """
    if path is None:
        path = []

    if isinstance(obj, CrossProd):
        return [(path, obj)]

    cprod_paths = []


class Distribution(ABC):
    """Generic Class to identify that this is a distribution"""

    def sample(self):
        pass

class Choice(Distribution):
    """Use to specify that this will be sampled from a list of elements"""

    def __init__(self, elements, p=None):
        self.elements = elements
        self.p = p

    def sample(self, rng=np.random):
        return rng.choice(self.elements, p=self.p)

def _find_cprod_with_paths(obj, path=None):
    """Find all the CrossProd objects and their paths in the given object.

    Args:
        obj (Any): The object to search for CrossProd objects.
        path (List[str]): The path to the current object.

    Returns:
        List[Tuple[List[str], CrossProd]]: A list of tuples where the first element is the path to the CrossProd
            object and the second element is the CrossProd object.
    """
    if path is None:
        path = []

    if isinstance(obj, CrossProd):
        return [(path, obj)]

    cprod_paths = []
    if is_dataclass(obj):
        for field in fields(obj):
            field_value = getattr(obj, field.name)
            cprod_paths += _find_cprod_with_paths(field_value, path + [field.name])
    elif isinstance(obj, dict):
        for key, value in obj.items():
            cprod_paths += _find_cprod_with_paths(value, path + [key])

    return cprod_paths

def _set_value(obj, path, value):
    """Set the value of the given path in the given object to the given value."""
    for key in path[:-1]:
        if isinstance(obj, dict):
            obj = obj[key]
        else:
            obj = getattr(obj, key)
    if isinstance(obj, dict):
        obj[path[-1]] = value
    else:
        setattr(obj, path[-1], value)

def expand_cross_product(obj: Any | list[Any]):
    """Expand the given object into a list of objects with all combinations of
    CrossProd objects.

    This function will recursively search for CrossProd objects in the given


            object and the second element is the CrossProd object.
    """
    if path is None:
        path = []

    if isinstance(obj, CrossProd):
        return [(path, obj)]

    cprod_paths = []
    if is_dataclass(obj):
        for field in fields(obj):
            field_value = getattr(obj, field.name)
            cprod_paths += _find_cprod_with_paths(field_value, path + [field.name])
    elif isinstance(obj, dict):
        for key, value in obj.items():
            cprod_paths += _find_cprod_with_paths(value, path + [key])

    return cprod_paths

def _set_value(obj, path, value):
    """Set the value of the given path in the given object to the given value."""
    for key in path[:-1]:
        if isinstance(obj, dict):
            obj = obj[key]
        else:
            obj = getattr(obj, key)
    if isinstance(obj, dict):
        obj[path[-1]] = value
    else:
        setattr(obj, path[-1], value)

def expand_cross_product(obj: Any | list[Any]):
    """Expand the given object into a list of objects with all combinations of
    CrossProd objects.

    This function will recursively search for CrossProd objects in the given
    object and create a list of objects with all combinations of CrossProd. It
    searches through dataclasses and dictionaries.

    Args:
        obj: Any | List[Any],
            The object to expand.

    Returns:
        List[Any]:
        A list of objects with all combinations of CrossProd objects.

    """


    elif isinstance(obj, dict):
        for key, value in obj.items():
            cprod_paths += _find_cprod_with_paths(value, path + [key])

    return cprod_paths

def _set_value(obj, path, value):
    """Set the value of the given path in the given object to the given value."""
    for key in path[:-1]:
        if isinstance(obj, dict):
            obj = obj[key]
        else:
            obj = getattr(obj, key)
    if isinstance(obj, dict):
        obj[path[-1]] = value
    else:
        setattr(obj, path[-1], value)

def expand_cross_product(obj: Any | list[Any]):
    """Expand the given object into a list of objects with all combinations of
    CrossProd objects.

    This function will recursively search for CrossProd objects in the given
    object and create a list of objects with all combinations of CrossProd. It
    searches through dataclasses and dictionaries.

    Args:
        obj: Any | List[Any],
            The object to expand.

    Returns:
        List[Any]:
        A list of objects with all combinations of CrossProd objects.

    """

    if isinstance(obj, CrossProd):
        return obj.elements

    if isinstance(obj, list):
        obj_list = obj
    else:
        obj_list = [obj]

    result = []

    for obj in obj_list:
        cprod_paths = _find_cprod_with_paths(obj)
        if not cprod_paths:
            result.append(copy.deepcopy(obj))
            continue

        paths, cprod_objects = zip(*cprod_paths)
        combinations = product(*[cprod_obj.elements for cprod_obj in cprod_objects])

        # create a base object with empty fields to make fast deep copies from
        base_obj = copy.deepcopy(obj)
        for path in paths:
            _set_value(base_obj, path, None)

        for combo in combinations:
            new_obj = copy.deepcopy(base_obj)
            for path, value in zip(paths, combo):
                _set_value(new_obj, path, value)
            result.append(new_obj)

    return result

def sample_and_expand_cross_product(obj: Any | list[Any], n_samples: int):
    """This will sample first and then expand the cross product."""
    return expand_cross_product(sample_args(obj, n_samples))

def sample_args(obj: Any | list[Any], n_samples: int):
    """Sample the given object n_samples times. Each sample is a deep copy of
    the original object with any object of type Distribution replaced by a
    sample from that distribution.

    Args:
        obj: Any | List[Any],
            the object to sample
        n_samples: int,
            the number of samples to generate

    Returns:
        List[Any]:


            result.append(copy.deepcopy(obj))
            continue

        paths, cprod_objects = zip(*cprod_paths)
        combinations = product(*[cprod_obj.elements for cprod_obj in cprod_objects])

        # create a base object with empty fields to make fast deep copies from
        base_obj = copy.deepcopy(obj)
        for path in paths:
            _set_value(base_obj, path, None)

        for combo in combinations:
            new_obj = copy.deepcopy(base_obj)
            for path, value in zip(paths, combo):
                _set_value(new_obj, path, value)
            result.append(new_obj)

    return result

def sample_and_expand_cross_product(obj: Any | list[Any], n_samples: int):
    """This will sample first and then expand the cross product."""
    return expand_cross_product(sample_args(obj, n_samples))

def sample_args(obj: Any | list[Any], n_samples: int):
    """Sample the given object n_samples times. Each sample is a deep copy of
    the original object with any object of type Distribution replaced by a
    sample from that distribution.

    Args:
        obj: Any | List[Any],
            the object to sample
        n_samples: int,
            the number of samples to generate

    Returns:
        List[Any]:
        A list of n_samples objects with the given object.
    """

    if isinstance(obj, list):
        obj_list = obj


        # create a base object with empty fields to make fast deep copies from
        base_obj = copy.deepcopy(obj)
        for path in paths:
            _set_value(base_obj, path, None)

        for combo in combinations:
            new_obj = copy.deepcopy(base_obj)
            for path, value in zip(paths, combo):
                _set_value(new_obj, path, value)
            result.append(new_obj)

    return result

def sample_and_expand_cross_product(obj: Any | list[Any], n_samples: int):
    """This will sample first and then expand the cross product."""
    return expand_cross_product(sample_args(obj, n_samples))

def sample_args(obj: Any | list[Any], n_samples: int):
    """Sample the given object n_samples times. Each sample is a deep copy of
    the original object with any object of type Distribution replaced by a
    sample from that distribution.

    Args:
        obj: Any | List[Any],
            the object to sample
        n_samples: int,
            the number of samples to generate

    Returns:
        List[Any]:
        A list of n_samples objects with the given object.
    """

    if isinstance(obj, list):
        obj_list = obj
    else:
        obj_list = [obj]

    result = []

    for obj in obj_list:
        for _ in range(n_samples):
            result.append(_sample_single(copy.deepcopy(obj)))

    return result

def _sample_single(obj):
    """Sample the given object once."""
    if isinstance(obj, Distribution):
        return obj.sample()

    if is_dataclass(obj):
        for field in fields(obj):
            value = getattr(obj, field.name)
            sampled_value = _sample_single(value)
            setattr(obj, field.name, sampled_value)
    elif isinstance(obj, dict):
        for key, value in obj.items():
            obj[key] = _sample_single(value)

    return obj

class Toggle:


    Returns:
        List[Any]:
        A list of n_samples objects with the given object.
    """

    if isinstance(obj, list):
        obj_list = obj
    else:
        obj_list = [obj]

    result = []

    for obj in obj_list:
        for _ in range(n_samples):
            result.append(_sample_single(copy.deepcopy(obj)))

    return result

def _sample_single(obj):
    """Sample the given object once."""
    if isinstance(obj, Distribution):
        return obj.sample()

    if is_dataclass(obj):
        for field in fields(obj):
            value = getattr(obj, field.name)
            sampled_value = _sample_single(value)
            setattr(obj, field.name, sampled_value)
    elif isinstance(obj, dict):
        for key, value in obj.items():
            obj[key] = _sample_single(value)

    return obj

class Toggle:
    pass

TOGGLE = Toggle()

def _change_value(obj, path, value):
    """Set the value to the given path in the nested objects.

    Note: This doesn't work with list or dict or tuples. Only works with objects.

    Args:
        obj: the object to change the value
        path: the path to the value to change
        value: the value to change to

    Raises:


Toggle

    return result

def _sample_single(obj):
    """Sample the given object once."""
    if isinstance(obj, Distribution):
        return obj.sample()

    if is_dataclass(obj):
        for field in fields(obj):
            value = getattr(obj, field.name)
            sampled_value = _sample_single(value)
            setattr(obj, field.name, sampled_value)
    elif isinstance(obj, dict):
        for key, value in obj.items():
            obj[key] = _sample_single(value)

    return obj

class Toggle:
    pass

TOGGLE = Toggle()

def _change_value(obj, path, value):
    """Set the value to the given path in the nested objects.

    Note: This doesn't work with list or dict or tuples. Only works with objects.

    Args:
        obj: the object to change the value
        path: the path to the value to change
        value: the value to change to

    Raises:
        ValueError: if the field is not found in the object
    """
    key_list = path.split(".")
    for key in key_list[:-1]:


    if is_dataclass(obj):
        for field in fields(obj):
            value = getattr(obj, field.name)
            sampled_value = _sample_single(value)
            setattr(obj, field.name, sampled_value)
    elif isinstance(obj, dict):
        for key, value in obj.items():
            obj[key] = _sample_single(value)

    return obj

class Toggle:
    pass

TOGGLE = Toggle()

def _change_value(obj, path, value):
    """Set the value to the given path in the nested objects.

    Note: This doesn't work with list or dict or tuples. Only works with objects.

    Args:
        obj: the object to change the value
        path: the path to the value to change
        value: the value to change to

    Raises:
        ValueError: if the field is not found in the object
    """
    key_list = path.split(".")
    for key in key_list[:-1]:
        if key == "":
            continue
        obj = getattr(obj, key)
    # if dataclass, then set the value only if the field exists

    def _set(obj, key, value):
        if isinstance(value, Toggle):
            previous_value = getattr(obj, key)
            if not isinstance(previous_value, bool):
                raise ValueError(f"Toggle object {obj} attribute {key} is not a boolean")
            setattr(obj, key, not previous_value)
        else:
            setattr(obj, key_list[-1], value)

    if is_dataclass(obj):
        field_names = [field.name for field in fields(obj)]
        if key_list[-1] in field_names:
            _set(obj, key_list[-1], value)
        else:
            raise ValueError(f"field {key_list[-1]} not found in {obj}")
    else:
        _set(obj, key_list[-1], value)

def _apply_change(params, change):
    """Apply the change to the params object in place."""
    if callable(change):
        change(params)
    elif isinstance(change, (tuple, list)):
        if isinstance(change[0], str) and len(change) == 2:
            path, value = change
            _change_value(params, path, value)
        else:
            for c in change:
                _apply_change(params, c)
    else:
        raise ValueError(f"change {change} not recognized")
    return params

def make_progression_study(start_point, changes, return_cross_prod=True):
    """A kind of ablation study by changing the start_point with changes one by


    def _set(obj, key, value):
        if isinstance(value, Toggle):
            previous_value = getattr(obj, key)
            if not isinstance(previous_value, bool):
                raise ValueError(f"Toggle object {obj} attribute {key} is not a boolean")
            setattr(obj, key, not previous_value)
        else:
            setattr(obj, key_list[-1], value)

    if is_dataclass(obj):
        field_names = [field.name for field in fields(obj)]
        if key_list[-1] in field_names:
            _set(obj, key_list[-1], value)
        else:
            raise ValueError(f"field {key_list[-1]} not found in {obj}")
    else:
        _set(obj, key_list[-1], value)

def _apply_change(params, change):
    """Apply the change to the params object in place."""
    if callable(change):
        change(params)
    elif isinstance(change, (tuple, list)):
        if isinstance(change[0], str) and len(change) == 2:
            path, value = change
            _change_value(params, path, value)
        else:
            for c in change:
                _apply_change(params, c)
    else:
        raise ValueError(f"change {change} not recognized")
    return params

def make_progression_study(start_point, changes, return_cross_prod=True):
    """A kind of ablation study by changing the start_point with changes one by
    one.

    Args:
        start_point: the starting point of the study
        changes: a list of changes to make to the start_point. Each change is
            either a callable or tuple containing a string identifying the path in the object to
            change and the value to change to. ex: (".obs.use_html", True)
        return_cross_prod: return a CrossProd object or just the list

    Returns:
        A CrossProd object containing a list of objects with progressive changes
        from `start_point`. If return_cross_prod is False, then it will return a
        list.

    """
    params_list = [start_point]


    else:
        _set(obj, key_list[-1], value)

def _apply_change(params, change):
    """Apply the change to the params object in place."""
    if callable(change):
        change(params)
    elif isinstance(change, (tuple, list)):
        if isinstance(change[0], str) and len(change) == 2:
            path, value = change
            _change_value(params, path, value)
        else:
            for c in change:
                _apply_change(params, c)
    else:
        raise ValueError(f"change {change} not recognized")
    return params

def make_progression_study(start_point, changes, return_cross_prod=True):
    """A kind of ablation study by changing the start_point with changes one by
    one.

    Args:
        start_point: the starting point of the study
        changes: a list of changes to make to the start_point. Each change is
            either a callable or tuple containing a string identifying the path in the object to
            change and the value to change to. ex: (".obs.use_html", True)
        return_cross_prod: return a CrossProd object or just the list

    Returns:
        A CrossProd object containing a list of objects with progressive changes
        from `start_point`. If return_cross_prod is False, then it will return a
        list.

    """
    params_list = [start_point]
    for change in changes:
        params = copy.deepcopy(params_list[-1])
        _apply_change(params, change)
        params_list.append(params)

    if return_cross_prod:
        return CrossProd(params_list)
    else:
        return params_list

def make_ablation_study(start_point, changes, return_cross_prod=True):
    """Ablation study by modifying the start_point with only one change at a
    time, and restarting from the original start_point for each configuration.

    Args:
        start_point: the starting point of the ablation study
        changes: a list of changes to make to the start_point. Each change is
            either a callable or tuple containing a string identifying the path in the object to
            change and the value to change to. ex: (".obs.use_html", True)
        return_cross_prod: return a CrossProd object or just the list

    Returns:
        A CrossProd object containing a list of objects with one change
        from `start_point`. If return_cross_prod is False, then it will return a
        list.

    """
    params_list = [start_point]


        return_cross_prod: return a CrossProd object or just the list

    Returns:
        A CrossProd object containing a list of objects with progressive changes
        from `start_point`. If return_cross_prod is False, then it will return a
        list.

    """
    params_list = [start_point]
    for change in changes:
        params = copy.deepcopy(params_list[-1])
        _apply_change(params, change)
        params_list.append(params)

    if return_cross_prod:
        return CrossProd(params_list)
    else:
        return params_list

def make_ablation_study(start_point, changes, return_cross_prod=True):
    """Ablation study by modifying the start_point with only one change at a
    time, and restarting from the original start_point for each configuration.

    Args:
        start_point: the starting point of the ablation study
        changes: a list of changes to make to the start_point. Each change is
            either a callable or tuple containing a string identifying the path in the object to
            change and the value to change to. ex: (".obs.use_html", True)
        return_cross_prod: return a CrossProd object or just the list

    Returns:
        A CrossProd object containing a list of objects with one change
        from `start_point`. If return_cross_prod is False, then it will return a
        list.

    """
    params_list = [start_point]
    for change in changes:
        params = copy.deepcopy(start_point)
        _apply_change(params, change)
        params_list.append(params)

    if return_cross_prod:
        return CrossProd(params_list)
    else:
        return params_list

if __name__ == "__main__":
    from agentlab.agents.dynamic_prompting import Flags

    study = make_progression_study(
        start_point=Flags(),
        changes=[
            ("use_thinking", True),
            (".use_plan", True),
            (".use_criticise", True),
        ],
        return_cross_prod=True,
    )

    study = expand_cross_product(study)
    for p in study:
        print(p)


import numpy as np

class CrossProd:
    """Use to specify that this will be part of a cross product"""

    def __init__(self, elements):
        self.elements = elements

class Distribution(ABC):
    """Generic Class to identify that this is a distribution"""

    def sample(self):
        pass

class Choice(Distribution):
    """Use to specify that this will be sampled from a list of elements"""

    def __init__(self, elements, p=None):
        self.elements = elements
        self.p = p

    def sample(self, rng=np.random):
        return rng.choice(self.elements, p=self.p)

def _find_cprod_with_paths(obj, path=None):
    """Find all the CrossProd objects and their paths in the given object.

    Args:
        obj (Any): The object to search for CrossProd objects.
        path (List[str]): The path to the current object.

    Returns:
        List[Tuple[List[str], CrossProd]]: A list of tuples where the first element is the path to the CrossProd
            object and the second element is the CrossProd object.
    """
    if path is None:
        path = []

    if isinstance(obj, CrossProd):


sample

    """Use to specify that this will be part of a cross product"""

    def __init__(self, elements):
        self.elements = elements

class Distribution(ABC):
    """Generic Class to identify that this is a distribution"""

    def sample(self):
        pass

class Choice(Distribution):
    """Use to specify that this will be sampled from a list of elements"""

    def __init__(self, elements, p=None):
        self.elements = elements
        self.p = p

    def sample(self, rng=np.random):
        return rng.choice(self.elements, p=self.p)

def _find_cprod_with_paths(obj, path=None):
    """Find all the CrossProd objects and their paths in the given object.

    Args:
        obj (Any): The object to search for CrossProd objects.
        path (List[str]): The path to the current object.

    Returns:
        List[Tuple[List[str], CrossProd]]: A list of tuples where the first element is the path to the CrossProd
            object and the second element is the CrossProd object.
    """
    if path is None:
        path = []

    if isinstance(obj, CrossProd):
        return [(path, obj)]

    cprod_paths = []


def _change_value(obj, path, value):
    """Set the value to the given path in the nested objects.

    Note: This doesn't work with list or dict or tuples. Only works with objects.

    Args:
        obj: the object to change the value
        path: the path to the value to change
        value: the value to change to

    Raises:
        ValueError: if the field is not found in the object
    """
    key_list = path.split(".")
    for key in key_list[:-1]:
        if key == "":
            continue
        obj = getattr(obj, key)
    # if dataclass, then set the value only if the field exists

    def _set(obj, key, value):
        if isinstance(value, Toggle):
            previous_value = getattr(obj, key)
            if not isinstance(previous_value, bool):
                raise ValueError(f"Toggle object {obj} attribute {key} is not a boolean")
            setattr(obj, key, not previous_value)
        else:
            setattr(obj, key_list[-1], value)

    if is_dataclass(obj):
        field_names = [field.name for field in fields(obj)]
        if key_list[-1] in field_names:
            _set(obj, key_list[-1], value)
        else:
            raise ValueError(f"field {key_list[-1]} not found in {obj}")
    else:
        _set(obj, key_list[-1], value)

def _apply_change(params, change):
    """Apply the change to the params object in place."""
    if callable(change):
        change(params)
    elif isinstance(change, (tuple, list)):
        if isinstance(change[0], str) and len(change) == 2:
            path, value = change
            _change_value(params, path, value)
        else:


import pandas as pd
from openai import OpenAI

if __name__ == "__main__":
    models = OpenAI().models.list()
    df = pd.DataFrame([dict(model) for model in models.data])

    # Filter GPT models or o1 models
    # df = df[df["id"].str.contains("gpt") | df["id"].str.contains("o1")]

    # Convert Unix timestamps to dates (YYYY-MM-DD) and remove time
    df["created"] = pd.to_datetime(df["created"], unit="s").dt.date
    df.sort_values(by="created", inplace=True)
    # Print all entries

    # print all entries
    print(df.to_string(index=False))