    # df.sort_index(inplace=True)

    sub_df = df.loc[row.name]
    orders = [get_exp_result(exp_dir).exp_args.order for exp_dir in sub_df.exp_dir]
    orders = [order for order in orders if order is not None]
    if len(orders) == 0:
        return None
    return np.mean(orders)

def _sort_order(result_df, report):
    """Add a column to the report with the average order for each agent and sort."""

    def add_order(row):
        return _get_avg_order(result_df, row)

    report["avg_order"] = report.apply(add_order, axis=1)
    return report.sort_values("avg_order", ascending=True)

def global_report(
    result_df: pd.DataFrame,
    reduce_fn=summarize,
    rename_index=lambda name: name.replace("agent.flags.", ""),
):
    """Produce a report that summarize all tasks and all episodes for each
    agent.

    Args:
        result_df: The result dataframe as returned by load_result_df.
        reduce_fn: The function to use to reduce the sub dataframe. By default
            this is summarize.
        rename_index: Function to rename the index. By default we remove the prefix
            "agent.flags."

    Returns:
        pd.DataFrame: The report
    """

    levels = list(range(result_df.index.nlevels))

    if len(levels) == 1:
        print("Only one configuration is found, returning a per-task report.")
        report = report_2d(result_df, reduce_fn=reduce_fn)
        report.loc["[ALL TASKS]"] = reduce_fn(result_df)
    else:
        print(
            "Found multiple configuration, averaging across tasks and returning a per-agent report."
        )
        report = result_df.groupby(level=levels[1:]).apply(reduce_fn)

        if rename_index is not None:
            index_names = [rename_index(name) for name in report.index.names]
            report = report.rename_axis(index=index_names)

        # if has key avg_reward
        if "avg_reward" in report.columns:
            report = report.sort_values("avg_reward", ascending=False)

    return report

def _rename_bool_flags(report: pd.DataFrame, true_str="✓", false_str="-"):
    """Rename the boolean flags to be more compact and readable."""
    map_bool = lambda x: true_str if x is True else false_str if x is False else x
    if isinstance(report.index, pd.MultiIndex):
        report.index = report.index.set_levels(
            [[map_bool(i) for i in level] for level in report.index.levels]
        )
    return report

def flag_report(report: pd.DataFrame, metric: str = "avg_reward", round_digits: int = 2):
    # for all index in the multi-index with boolean value, get the average for
    # True and the average for False separately. Produce a new dataframe with
    # the average for True and False for each index and the ratio between the
    # two as a new column.

    # check the number of levels
    if report.index.nlevels <= 1:


        print("Only one configuration is found, returning a per-task report.")
        report = report_2d(result_df, reduce_fn=reduce_fn)
        report.loc["[ALL TASKS]"] = reduce_fn(result_df)
    else:
        print(
            "Found multiple configuration, averaging across tasks and returning a per-agent report."
        )
        report = result_df.groupby(level=levels[1:]).apply(reduce_fn)

        if rename_index is not None:
            index_names = [rename_index(name) for name in report.index.names]
            report = report.rename_axis(index=index_names)

        # if has key avg_reward
        if "avg_reward" in report.columns:
            report = report.sort_values("avg_reward", ascending=False)

    return report

def _rename_bool_flags(report: pd.DataFrame, true_str="✓", false_str="-"):
    """Rename the boolean flags to be more compact and readable."""
    map_bool = lambda x: true_str if x is True else false_str if x is False else x
    if isinstance(report.index, pd.MultiIndex):
        report.index = report.index.set_levels(
            [[map_bool(i) for i in level] for level in report.index.levels]
        )
    return report

def flag_report(report: pd.DataFrame, metric: str = "avg_reward", round_digits: int = 2):
    # for all index in the multi-index with boolean value, get the average for
    # True and the average for False separately. Produce a new dataframe with
    # the average for True and False for each index and the ratio between the
    # two as a new column.

    # check the number of levels
    if report.index.nlevels <= 1:
        print(f"Only {report.index.nlevels} levels in the index, cannot produce flag report.")
        return

    report = report.copy()
    report = report.reset_index()

    records = []
    for col in report.columns:
        if report[col].dtype == bool:
            avg_true = report[report[col]][metric].mean()


            index_names = [rename_index(name) for name in report.index.names]
            report = report.rename_axis(index=index_names)

        # if has key avg_reward
        if "avg_reward" in report.columns:
            report = report.sort_values("avg_reward", ascending=False)

    return report

def _rename_bool_flags(report: pd.DataFrame, true_str="✓", false_str="-"):
    """Rename the boolean flags to be more compact and readable."""
    map_bool = lambda x: true_str if x is True else false_str if x is False else x
    if isinstance(report.index, pd.MultiIndex):
        report.index = report.index.set_levels(
            [[map_bool(i) for i in level] for level in report.index.levels]
        )
    return report

def flag_report(report: pd.DataFrame, metric: str = "avg_reward", round_digits: int = 2):
    # for all index in the multi-index with boolean value, get the average for
    # True and the average for False separately. Produce a new dataframe with
    # the average for True and False for each index and the ratio between the
    # two as a new column.

    # check the number of levels
    if report.index.nlevels <= 1:
        print(f"Only {report.index.nlevels} levels in the index, cannot produce flag report.")
        return

    report = report.copy()
    report = report.reset_index()

    records = []
    for col in report.columns:
        if report[col].dtype == bool:
            avg_true = report[report[col]][metric].mean()
            avg_false = report[~report[col]][metric].mean()
            ratio = avg_true / avg_false
            records.append(dict(hparam=col, avg_true=avg_true, avg_false=avg_false, ratio=ratio))

    flag_report = pd.DataFrame(records).set_index("hparam")
    flag_report.sort_values("ratio", ascending=False, inplace=True)
    if round_digits is not None:
        flag_report = flag_report.round(round_digits)

    return flag_report

def display_report(
    report: pd.DataFrame,
    apply_shrink_columns: bool = True,
    copy_to_clipboard: bool = True,
    rename_bool_flags: bool = True,
    print_only: str = None,
):
    """Display the report in a nicer-ish format.

    To be able to wrap col names we need to use set_wrap_stype, which returns a
    styled df, and doesn't behave like a normal df. For encapsulate the displaying in
    this function.

    Args:
        report: The report to display
        apply_shrink_columns: Make the column more compat by replacing
            underscores with newlines
        copy_to_clipboard: Copy the report to the clipboard


    report = report.copy()
    report = report.reset_index()

    records = []
    for col in report.columns:
        if report[col].dtype == bool:
            avg_true = report[report[col]][metric].mean()
            avg_false = report[~report[col]][metric].mean()
            ratio = avg_true / avg_false
            records.append(dict(hparam=col, avg_true=avg_true, avg_false=avg_false, ratio=ratio))

    flag_report = pd.DataFrame(records).set_index("hparam")
    flag_report.sort_values("ratio", ascending=False, inplace=True)
    if round_digits is not None:
        flag_report = flag_report.round(round_digits)

    return flag_report

def display_report(
    report: pd.DataFrame,
    apply_shrink_columns: bool = True,
    copy_to_clipboard: bool = True,
    rename_bool_flags: bool = True,
    print_only: str = None,
):
    """Display the report in a nicer-ish format.

    To be able to wrap col names we need to use set_wrap_stype, which returns a
    styled df, and doesn't behave like a normal df. For encapsulate the displaying in
    this function.

    Args:
        report: The report to display
        apply_shrink_columns: Make the column more compat by replacing
            underscores with newlines
        copy_to_clipboard: Copy the report to the clipboard
        rename_bool_flags: Rename the boolean flags to be more compact and readable
        print_only: Print only the given column
    """
    report = report.copy()

    if apply_shrink_columns:
        report = shrink_columns(report)

    if rename_bool_flags:
        report = _rename_bool_flags(report)

    # if copy_to_clipboard:
    #     to_clipboard(report)

    columns = list(report.columns)

    report.reset_index(inplace=True)

    if print_only:
        columns = [print_only] + columns
        report = report[columns]

    styled_report = set_wrap_style(report)

    display(styled_report)

def shrink_columns(df, also_wrap_index=True):
    """Make the column names more compact by replacing underscores with newlines"""
    df = df.copy()

    df.columns = [col.replace("_", "\n") for col in df.columns]
    if also_wrap_index:
        df.index.names = [name.replace("_", "\n") for name in df.index.names]

    # Define a formatter function that formats float numbers without trailing zeros
    def formatter(x):
        if isinstance(x, float):
            return "{:.10f}".format(x).rstrip("0").rstrip(".")
        return x

    return df.map(formatter)

def set_wrap_style(df):


    if rename_bool_flags:
        report = _rename_bool_flags(report)

    # if copy_to_clipboard:
    #     to_clipboard(report)

    columns = list(report.columns)

    report.reset_index(inplace=True)

    if print_only:
        columns = [print_only] + columns
        report = report[columns]

    styled_report = set_wrap_style(report)

    display(styled_report)

def shrink_columns(df, also_wrap_index=True):
    """Make the column names more compact by replacing underscores with newlines"""
    df = df.copy()

    df.columns = [col.replace("_", "\n") for col in df.columns]
    if also_wrap_index:
        df.index.names = [name.replace("_", "\n") for name in df.index.names]

    # Define a formatter function that formats float numbers without trailing zeros
    def formatter(x):
        if isinstance(x, float):
            return "{:.10f}".format(x).rstrip("0").rstrip(".")
        return x

    return df.map(formatter)

def set_wrap_style(df):
    return df.style.set_table_styles([{"selector": "th", "props": [("white-space", "pre-wrap")]}])

# ------------
# Error Utils
# ------------

def map_err_key(err_msg: str):
    if err_msg is None:
        return err_msg

    # remove logs from the message if any
    err_msg = err_msg[: err_msg.find("=== logs ===")].rstrip()
    regex_replacements = [
        (
            r"your messages resulted in \d+ tokens",


    display(styled_report)

def shrink_columns(df, also_wrap_index=True):
    """Make the column names more compact by replacing underscores with newlines"""
    df = df.copy()

    df.columns = [col.replace("_", "\n") for col in df.columns]
    if also_wrap_index:
        df.index.names = [name.replace("_", "\n") for name in df.index.names]

    # Define a formatter function that formats float numbers without trailing zeros
    def formatter(x):
        if isinstance(x, float):
            return "{:.10f}".format(x).rstrip("0").rstrip(".")
        return x

    return df.map(formatter)

def set_wrap_style(df):
    return df.style.set_table_styles([{"selector": "th", "props": [("white-space", "pre-wrap")]}])

# ------------
# Error Utils
# ------------

def map_err_key(err_msg: str):
    if err_msg is None:
        return err_msg

    # remove logs from the message if any
    err_msg = err_msg[: err_msg.find("=== logs ===")].rstrip()
    regex_replacements = [
        (
            r"your messages resulted in \d+ tokens",
            "your messages resulted in x tokens",
        ),
        (
            r"(?<=Exception uncaught by agent or environment in task\s)([^\s]+)",


        df.index.names = [name.replace("_", "\n") for name in df.index.names]

    # Define a formatter function that formats float numbers without trailing zeros
    def formatter(x):
        if isinstance(x, float):
            return "{:.10f}".format(x).rstrip("0").rstrip(".")
        return x

    return df.map(formatter)

def set_wrap_style(df):
    return df.style.set_table_styles([{"selector": "th", "props": [("white-space", "pre-wrap")]}])

# ------------
# Error Utils
# ------------

def map_err_key(err_msg: str):
    if err_msg is None:
        return err_msg

    # remove logs from the message if any
    err_msg = err_msg[: err_msg.find("=== logs ===")].rstrip()
    regex_replacements = [
        (
            r"your messages resulted in \d+ tokens",
            "your messages resulted in x tokens",
        ),
        (
            r"(?<=Exception uncaught by agent or environment in task\s)([^\s]+)",
            "<task_name>.",
        ),
    ]

    for pattern, replacement in regex_replacements:
        err_msg = re.sub(pattern, replacement, err_msg)
    return err_msg

def error_report(df: pd.DataFrame, max_stack_trace=10, use_log=False):
    """Report the error message for each agent."""

    if "err_key" not in df:
        df["err_key"] = df["err_msg"].map(map_err_key)

    unique_counts = df["err_key"].value_counts().sort_values(ascending=False)
    report = []
    for err_key, count in unique_counts.items():
        report.append("-------------------")
        report.append(f"## {count}x : " + err_key.replace("\n", "<br>") + "\n")

        # find sub_df with this error message
        sub_df = df[df["err_key"] == err_key]
        idx = 0

        exp_result_list = [get_exp_result(row.exp_dir) for _, row in sub_df.iterrows()]
        exp_result_list = sorted(exp_result_list, key=lambda x: x.exp_args.env_args.task_name)


        return err_msg

    # remove logs from the message if any
    err_msg = err_msg[: err_msg.find("=== logs ===")].rstrip()
    regex_replacements = [
        (
            r"your messages resulted in \d+ tokens",
            "your messages resulted in x tokens",
        ),
        (
            r"(?<=Exception uncaught by agent or environment in task\s)([^\s]+)",
            "<task_name>.",
        ),
    ]

    for pattern, replacement in regex_replacements:
        err_msg = re.sub(pattern, replacement, err_msg)
    return err_msg

def error_report(df: pd.DataFrame, max_stack_trace=10, use_log=False):
    """Report the error message for each agent."""

    if "err_key" not in df:
        df["err_key"] = df["err_msg"].map(map_err_key)

    unique_counts = df["err_key"].value_counts().sort_values(ascending=False)
    report = []
    for err_key, count in unique_counts.items():
        report.append("-------------------")
        report.append(f"## {count}x : " + err_key.replace("\n", "<br>") + "\n")

        # find sub_df with this error message
        sub_df = df[df["err_key"] == err_key]
        idx = 0

        exp_result_list = [get_exp_result(row.exp_dir) for _, row in sub_df.iterrows()]
        exp_result_list = sorted(exp_result_list, key=lambda x: x.exp_args.env_args.task_name)
        for exp_result in exp_result_list:
            report.append(
                f"* {exp_result.exp_args.env_args.task_name} seed: {exp_result.exp_args.env_args.task_seed}"
            )

        report.append(f"\nShowing Max {max_stack_trace} stack traces:\n")
        for exp_result in exp_result_list:
            if idx >= max_stack_trace:
                break

            if not use_log:
                # print task name and stack trace
                stack_trace = exp_result.summary_info.get("stack_trace", "")
                report.append(f"Task Name: {exp_result.exp_args.env_args.task_name}\n")
                report.append(f"exp_dir: {exp_result.exp_dir}\n")
                report.append(f"Stack Trace: \n {stack_trace}\n")
                report.append("\n")
            else:
                report.append(f"```bash\n{_format_log(exp_result)}\n```")

            idx += 1

    return "\n".join(report)

def _format_log(exp_result: ExpResult, head_lines=10, tail_lines=50):
    """Extract head and tail of the log. Try to find the traceback."""
    log = exp_result.logs
    if log is None:
        return "No log found"

    log_lines = log.split("\n")
    if len(log_lines) <= head_lines + tail_lines:
        return log

    # first 10 lines:
    log_head = "\n".join(log_lines[:head_lines])

    try:
        traceback_idx = log.rindex("Traceback (most recent call last):")
        tail_idx = log.rindex("action:", 0, traceback_idx)
        log_tail = log[tail_idx:]
    except ValueError:


        report.append(f"\nShowing Max {max_stack_trace} stack traces:\n")
        for exp_result in exp_result_list:
            if idx >= max_stack_trace:
                break

            if not use_log:
                # print task name and stack trace
                stack_trace = exp_result.summary_info.get("stack_trace", "")
                report.append(f"Task Name: {exp_result.exp_args.env_args.task_name}\n")
                report.append(f"exp_dir: {exp_result.exp_dir}\n")
                report.append(f"Stack Trace: \n {stack_trace}\n")
                report.append("\n")
            else:
                report.append(f"```bash\n{_format_log(exp_result)}\n```")

            idx += 1

    return "\n".join(report)

def _format_log(exp_result: ExpResult, head_lines=10, tail_lines=50):
    """Extract head and tail of the log. Try to find the traceback."""
    log = exp_result.logs
    if log is None:
        return "No log found"

    log_lines = log.split("\n")
    if len(log_lines) <= head_lines + tail_lines:
        return log

    # first 10 lines:
    log_head = "\n".join(log_lines[:head_lines])

    try:
        traceback_idx = log.rindex("Traceback (most recent call last):")
        tail_idx = log.rindex("action:", 0, traceback_idx)
        log_tail = log[tail_idx:]
    except ValueError:
        log_tail = "\n".join(log_lines[-tail_lines:])

    return log_head + "\n...\n...truncated middle of the log\n...\n" + log_tail

def categorize_error(row):
    if pd.isna(row.get("err_msg", None)):
        return None
    for category, check_function in ERR_CLASS_MAP.items():
        if check_function(row["err_msg"], row["stack_trace"]):
            if category == "critical_server_error":
                return is_critical_server_error(
                    row["err_msg"], row["stack_trace"], return_error_type=True
                )
            elif category == "minor_server_error":
                return is_minor_server_error(
                    row["err_msg"], row["stack_trace"], return_error_type=True
                )
            return category
    return "other_error"

def error_report_detailed(df: pd.DataFrame, max_stack_trace=10):


    if log is None:
        return "No log found"

    log_lines = log.split("\n")
    if len(log_lines) <= head_lines + tail_lines:
        return log

    # first 10 lines:
    log_head = "\n".join(log_lines[:head_lines])

    try:
        traceback_idx = log.rindex("Traceback (most recent call last):")
        tail_idx = log.rindex("action:", 0, traceback_idx)
        log_tail = log[tail_idx:]
    except ValueError:
        log_tail = "\n".join(log_lines[-tail_lines:])

    return log_head + "\n...\n...truncated middle of the log\n...\n" + log_tail

def categorize_error(row):
    if pd.isna(row.get("err_msg", None)):
        return None
    for category, check_function in ERR_CLASS_MAP.items():
        if check_function(row["err_msg"], row["stack_trace"]):
            if category == "critical_server_error":
                return is_critical_server_error(
                    row["err_msg"], row["stack_trace"], return_error_type=True
                )
            elif category == "minor_server_error":
                return is_minor_server_error(
                    row["err_msg"], row["stack_trace"], return_error_type=True
                )
            return category
    return "other_error"

def error_report_detailed(df: pd.DataFrame, max_stack_trace=10):
    """Report the error message for each agent, categorizing them as server errors or retry errors."""

    df["error_category"] = df.apply(categorize_error, axis=1)

    report = []
    for category in df["error_category"].unique():
        if category is None:
            continue
        report.append("\n-------------------")
        report.append(f"Category: {category}")
        report.append("-------------------\n")
        report.append(f"Total number of errors: {len(df[df['error_category'] == category])}\n")
        category_df = df[df["error_category"] == category]
        unique_counts = category_df["err_msg"].value_counts().sort_values(ascending=False)

        idx = 0
        for err_msg, count in unique_counts.items():


    return log_head + "\n...\n...truncated middle of the log\n...\n" + log_tail

def categorize_error(row):
    if pd.isna(row.get("err_msg", None)):
        return None
    for category, check_function in ERR_CLASS_MAP.items():
        if check_function(row["err_msg"], row["stack_trace"]):
            if category == "critical_server_error":
                return is_critical_server_error(
                    row["err_msg"], row["stack_trace"], return_error_type=True
                )
            elif category == "minor_server_error":
                return is_minor_server_error(
                    row["err_msg"], row["stack_trace"], return_error_type=True
                )
            return category
    return "other_error"

def error_report_detailed(df: pd.DataFrame, max_stack_trace=10):
    """Report the error message for each agent, categorizing them as server errors or retry errors."""

    df["error_category"] = df.apply(categorize_error, axis=1)

    report = []
    for category in df["error_category"].unique():
        if category is None:
            continue
        report.append("\n-------------------")
        report.append(f"Category: {category}")
        report.append("-------------------\n")
        report.append(f"Total number of errors: {len(df[df['error_category'] == category])}\n")
        category_df = df[df["error_category"] == category]
        unique_counts = category_df["err_msg"].value_counts().sort_values(ascending=False)

        idx = 0
        for err_msg, count in unique_counts.items():
            if idx >= max_stack_trace:
                break
            idx += 1
            report.append("-------------------")
            report.append(f"{count}x : {err_msg}\n")
            sub_df = category_df[category_df["err_msg"] == err_msg]
            for _, row in sub_df.iterrows():
                exp_result = ExpResult(row.exp_dir)
                report.append(f"Task Name: {exp_result.exp_args.env_args.task_name}\n")
                report.append(f"exp_dir: {exp_result.exp_dir}\n")
                report.append(f"Stack Trace: \n {row['stack_trace']}\n")
                report.append("\n")
                break

    return "\n".join(report)

def print_errors_chronologically(df: pd.DataFrame):
    """Print the errors in chronological order, grouping contiguous chunks of the same error."""
    df = df.sort_values("exp_date", ascending=True)

    current_error = None
    error_count = 0

    for _, row in df.iterrows():
        if pd.isna(row.get("err_msg", None)):
            continue

        error = categorize_error(row)

        if error != current_error:
            if current_error is not None:
                print(f"{current_error.ljust(40)} : {str(error_count).rjust(5)} times")

            current_error = error


        idx = 0
        for err_msg, count in unique_counts.items():
            if idx >= max_stack_trace:
                break
            idx += 1
            report.append("-------------------")
            report.append(f"{count}x : {err_msg}\n")
            sub_df = category_df[category_df["err_msg"] == err_msg]
            for _, row in sub_df.iterrows():
                exp_result = ExpResult(row.exp_dir)
                report.append(f"Task Name: {exp_result.exp_args.env_args.task_name}\n")
                report.append(f"exp_dir: {exp_result.exp_dir}\n")
                report.append(f"Stack Trace: \n {row['stack_trace']}\n")
                report.append("\n")
                break

    return "\n".join(report)

def print_errors_chronologically(df: pd.DataFrame):
    """Print the errors in chronological order, grouping contiguous chunks of the same error."""
    df = df.sort_values("exp_date", ascending=True)

    current_error = None
    error_count = 0

    for _, row in df.iterrows():
        if pd.isna(row.get("err_msg", None)):
            continue

        error = categorize_error(row)

        if error != current_error:
            if current_error is not None:
                print(f"{current_error.ljust(40)} : {str(error_count).rjust(5)} times")

            current_error = error
            error_count = 1
        else:
            error_count += 1

    if current_error is not None:
        print(f"{current_error.ljust(40)} : {str(error_count).rjust(5)} times")

def report_different_errors(sub_df):
    """Report the different errors in the dataframe."""

    def _categorize_error(row):
        if pd.isna(row.err_msg):
            record = {}
        else:
            record = {
                err_class: err_fn(row.err_msg, row.stack_trace)
                for err_class, err_fn in ERR_CLASS_MAP.items()
            }
            record["other_err"] = np.sum(list(record.values())) == 0
            record["any_err"] = True

        return pd.Series(record)

    error_report = sub_df.apply(_categorize_error, axis=1).sum(skipna=True)


    for _, row in df.iterrows():
        if pd.isna(row.get("err_msg", None)):
            continue

        error = categorize_error(row)

        if error != current_error:
            if current_error is not None:
                print(f"{current_error.ljust(40)} : {str(error_count).rjust(5)} times")

            current_error = error
            error_count = 1
        else:
            error_count += 1

    if current_error is not None:
        print(f"{current_error.ljust(40)} : {str(error_count).rjust(5)} times")

def report_different_errors(sub_df):
    """Report the different errors in the dataframe."""

    def _categorize_error(row):
        if pd.isna(row.err_msg):
            record = {}
        else:
            record = {
                err_class: err_fn(row.err_msg, row.stack_trace)
                for err_class, err_fn in ERR_CLASS_MAP.items()
            }
            record["other_err"] = np.sum(list(record.values())) == 0
            record["any_err"] = True

        return pd.Series(record)

    error_report = sub_df.apply(_categorize_error, axis=1).sum(skipna=True)

    # TODO: fix this bug
    assert isinstance(error_report, pd.DataFrame), "Expected a DataFrame, got a Series."

    return error_report

# ===============

def _benchmark_from_task_name(task_name: str):
    """Extract the benchmark from the task name."""
    # TODO should be more robost, e.g. handle workarna.L1, workarena.L2, etc.
    return task_name.split(".")[0]

def summarize_study(result_df: pd.DataFrame) -> pd.DataFrame:
    """Create a summary of the study. Similar to global report, but handles single agent differently."""

    levels = list(range(result_df.index.nlevels))
    return result_df.groupby(level=levels[1:]).apply(summarize)

def split_by_key(df: pd.DataFrame, key):
    """Return a dict of dataframes spearted by the given key."""


            record = {
                err_class: err_fn(row.err_msg, row.stack_trace)
                for err_class, err_fn in ERR_CLASS_MAP.items()
            }
            record["other_err"] = np.sum(list(record.values())) == 0
            record["any_err"] = True

        return pd.Series(record)

    error_report = sub_df.apply(_categorize_error, axis=1).sum(skipna=True)

    # TODO: fix this bug
    assert isinstance(error_report, pd.DataFrame), "Expected a DataFrame, got a Series."

    return error_report

# ===============

def _benchmark_from_task_name(task_name: str):
    """Extract the benchmark from the task name."""
    # TODO should be more robost, e.g. handle workarna.L1, workarena.L2, etc.
    return task_name.split(".")[0]

def summarize_study(result_df: pd.DataFrame) -> pd.DataFrame:
    """Create a summary of the study. Similar to global report, but handles single agent differently."""

    levels = list(range(result_df.index.nlevels))
    return result_df.groupby(level=levels[1:]).apply(summarize)

def split_by_key(df: pd.DataFrame, key):
    """Return a dict of dataframes spearted by the given key."""
    # check if key in df
    if key not in df.columns:
        df = df.reset_index(key, inplace=False)

    df_dict = {}
    for value in df[key].unique():
        sub_df = df[df[key] == value].copy()
        set_index_from_variables(sub_df)
        df_dict[value] = sub_df


        return pd.Series(record)

    error_report = sub_df.apply(_categorize_error, axis=1).sum(skipna=True)

    # TODO: fix this bug
    assert isinstance(error_report, pd.DataFrame), "Expected a DataFrame, got a Series."

    return error_report

# ===============

def _benchmark_from_task_name(task_name: str):
    """Extract the benchmark from the task name."""
    # TODO should be more robost, e.g. handle workarna.L1, workarena.L2, etc.
    return task_name.split(".")[0]

def summarize_study(result_df: pd.DataFrame) -> pd.DataFrame:
    """Create a summary of the study. Similar to global report, but handles single agent differently."""

    levels = list(range(result_df.index.nlevels))
    return result_df.groupby(level=levels[1:]).apply(summarize)

def split_by_key(df: pd.DataFrame, key):
    """Return a dict of dataframes spearted by the given key."""
    # check if key in df
    if key not in df.columns:
        df = df.reset_index(key, inplace=False)

    df_dict = {}
    for value in df[key].unique():
        sub_df = df[df[key] == value].copy()
        set_index_from_variables(sub_df)
        df_dict[value] = sub_df

    return df_dict

def get_all_summaries(results_dir: Path, skip_hidden=True, ignore_cache=False, ignore_stale=False):
    summaries = []
    for study_dir in results_dir.iterdir():


    return error_report

# ===============

def _benchmark_from_task_name(task_name: str):
    """Extract the benchmark from the task name."""
    # TODO should be more robost, e.g. handle workarna.L1, workarena.L2, etc.
    return task_name.split(".")[0]

def summarize_study(result_df: pd.DataFrame) -> pd.DataFrame:
    """Create a summary of the study. Similar to global report, but handles single agent differently."""

    levels = list(range(result_df.index.nlevels))
    return result_df.groupby(level=levels[1:]).apply(summarize)

def split_by_key(df: pd.DataFrame, key):
    """Return a dict of dataframes spearted by the given key."""
    # check if key in df
    if key not in df.columns:
        df = df.reset_index(key, inplace=False)

    df_dict = {}
    for value in df[key].unique():
        sub_df = df[df[key] == value].copy()
        set_index_from_variables(sub_df)
        df_dict[value] = sub_df

    return df_dict

def get_all_summaries(results_dir: Path, skip_hidden=True, ignore_cache=False, ignore_stale=False):
    summaries = []
    for study_dir in results_dir.iterdir():
        print(study_dir.name)
        if skip_hidden and study_dir.name.startswith("_"):
            print("  skip (starts with '_')")
            continue

        try:
            summary = get_study_summary(
                study_dir, ignore_cache=ignore_cache, ignore_stale=ignore_stale
            )
            if summary is not None:
                # set as index
                summary["study_dir"] = study_dir.name
                summary.set_index("study_dir", inplace=True)
                summaries.append(summary)


    levels = list(range(result_df.index.nlevels))
    return result_df.groupby(level=levels[1:]).apply(summarize)

def split_by_key(df: pd.DataFrame, key):
    """Return a dict of dataframes spearted by the given key."""
    # check if key in df
    if key not in df.columns:
        df = df.reset_index(key, inplace=False)

    df_dict = {}
    for value in df[key].unique():
        sub_df = df[df[key] == value].copy()
        set_index_from_variables(sub_df)
        df_dict[value] = sub_df

    return df_dict

def get_all_summaries(results_dir: Path, skip_hidden=True, ignore_cache=False, ignore_stale=False):
    summaries = []
    for study_dir in results_dir.iterdir():
        print(study_dir.name)
        if skip_hidden and study_dir.name.startswith("_"):
            print("  skip (starts with '_')")
            continue

        try:
            summary = get_study_summary(
                study_dir, ignore_cache=ignore_cache, ignore_stale=ignore_stale
            )
            if summary is not None:
                # set as index
                summary["study_dir"] = study_dir.name
                summary.set_index("study_dir", inplace=True)
                summaries.append(summary)

        except Exception:
            traceback.print_exc()
            continue

    summaries = pd.concat(summaries)
    # reverse sort according to index
    summaries.sort_index(ascending=False, inplace=True)
    return summaries

def get_study_summary(
    study_dir: Path,
    ignore_cache=False,
    ignore_stale=False,
    progress_fn=None,
    sentinel=None,
) -> pd.DataFrame:
    """Get the cached study summary for the given study directory or computes it.

    The cache is based on the modified times of all the files in the study.

    Args:
        study_dir: The study directory to summarize
        ignore_cache: If True, ignore the cache and recompute the summary
        ignore_stale: If True, don't verify if files have changed since the last
            summary was computed. This may lead to stale summaries.
        progress_fn: Pass tqdm.tqdm to show progress.
        sentinel: Captures internal values for unit testing.


        try:
            summary = get_study_summary(
                study_dir, ignore_cache=ignore_cache, ignore_stale=ignore_stale
            )
            if summary is not None:
                # set as index
                summary["study_dir"] = study_dir.name
                summary.set_index("study_dir", inplace=True)
                summaries.append(summary)

        except Exception:
            traceback.print_exc()
            continue

    summaries = pd.concat(summaries)
    # reverse sort according to index
    summaries.sort_index(ascending=False, inplace=True)
    return summaries

def get_study_summary(
    study_dir: Path,
    ignore_cache=False,
    ignore_stale=False,
    progress_fn=None,
    sentinel=None,
) -> pd.DataFrame:
    """Get the cached study summary for the given study directory or computes it.

    The cache is based on the modified times of all the files in the study.

    Args:
        study_dir: The study directory to summarize
        ignore_cache: If True, ignore the cache and recompute the summary
        ignore_stale: If True, don't verify if files have changed since the last
            summary was computed. This may lead to stale summaries.
        progress_fn: Pass tqdm.tqdm to show progress.
        sentinel: Captures internal values for unit testing.

    Returns:
        pd.DataFrame: The study summary
    """
    study_dir = Path(study_dir)

    summary_path = study_dir / "study_summary.csv"
    if not ignore_stale:
        is_stale = _is_stale(study_dir, summary_path)
    else:
        is_stale = False

    if not ignore_cache:
        if summary_path.exists() and not is_stale:
            if sentinel is not None:
                sentinel["from_cache"] = True
            return pd.read_csv(summary_path)

    result_df = load_result_df(study_dir, progress_fn=progress_fn)
    if result_df is None:
        return None

    summary = summarize_study(result_df)

    summary.to_csv(summary_path)

    if sentinel is not None:
        sentinel["from_cache"] = False
    return summary

def _get_mtimes(dir: Path, pattern="[!_.]*", whitelist=()):
    """Recursevly get all file's modif date"""
    # use glob to get all files
    files = list(dir.rglob(pattern))
    return {str(f.relative_to(dir)): f.stat().st_mtime for f in files if f not in whitelist}

def _is_stale(study_dir: Path, summary_path: Path) -> bool:
    mtimes_path = study_dir / "_last_modification_times.json"
    mtimes = _get_mtimes(study_dir, whitelist=(summary_path,))
    if not mtimes_path.exists() or not summary_path.exists():
        stale = True
    else:
        mtimes_saved = json.loads(mtimes_path.read_text())
        stale = mtimes_saved != mtimes
    mtimes_path.write_text(json.dumps(mtimes))
    return stale


    if not ignore_cache:
        if summary_path.exists() and not is_stale:
            if sentinel is not None:
                sentinel["from_cache"] = True
            return pd.read_csv(summary_path)

    result_df = load_result_df(study_dir, progress_fn=progress_fn)
    if result_df is None:
        return None

    summary = summarize_study(result_df)

    summary.to_csv(summary_path)

    if sentinel is not None:
        sentinel["from_cache"] = False
    return summary

def _get_mtimes(dir: Path, pattern="[!_.]*", whitelist=()):
    """Recursevly get all file's modif date"""
    # use glob to get all files
    files = list(dir.rglob(pattern))
    return {str(f.relative_to(dir)): f.stat().st_mtime for f in files if f not in whitelist}

def _is_stale(study_dir: Path, summary_path: Path) -> bool:
    mtimes_path = study_dir / "_last_modification_times.json"
    mtimes = _get_mtimes(study_dir, whitelist=(summary_path,))
    if not mtimes_path.exists() or not summary_path.exists():
        stale = True
    else:
        mtimes_saved = json.loads(mtimes_path.read_text())
        stale = mtimes_saved != mtimes
    mtimes_path.write_text(json.dumps(mtimes))
    return stale

def get_all_task_messages(exp_dir, max_n_exp=None):
    result_list = list(yield_all_exp_results(exp_dir, progress_fn=tqdm))

    if max_n_exp is not None:
        result_list = random.sample(result_list, min(max_n_exp, len(result_list)))


    result_df = load_result_df(study_dir, progress_fn=progress_fn)
    if result_df is None:
        return None

    summary = summarize_study(result_df)

    summary.to_csv(summary_path)

    if sentinel is not None:
        sentinel["from_cache"] = False
    return summary

def _get_mtimes(dir: Path, pattern="[!_.]*", whitelist=()):
    """Recursevly get all file's modif date"""
    # use glob to get all files
    files = list(dir.rglob(pattern))
    return {str(f.relative_to(dir)): f.stat().st_mtime for f in files if f not in whitelist}

def _is_stale(study_dir: Path, summary_path: Path) -> bool:
    mtimes_path = study_dir / "_last_modification_times.json"
    mtimes = _get_mtimes(study_dir, whitelist=(summary_path,))
    if not mtimes_path.exists() or not summary_path.exists():
        stale = True
    else:
        mtimes_saved = json.loads(mtimes_path.read_text())
        stale = mtimes_saved != mtimes
    mtimes_path.write_text(json.dumps(mtimes))
    return stale

def get_all_task_messages(exp_dir, max_n_exp=None):
    result_list = list(yield_all_exp_results(exp_dir, progress_fn=tqdm))

    if max_n_exp is not None:
        result_list = random.sample(result_list, min(max_n_exp, len(result_list)))

    task_messages = defaultdict(list)
    for exp_result in tqdm(result_list):
        task_name = exp_result.exp_args.env_args.task_name
        for step in exp_result.steps_info:
            try:
                task_messages[task_name].append(step.task_info["message"])
            except (KeyError, TypeError):
                pass

    # count identical task messages:
    for task_name, messages in task_messages.items():
        unique_messages, count = np.unique(messages, return_counts=True)


def _get_mtimes(dir: Path, pattern="[!_.]*", whitelist=()):
    """Recursevly get all file's modif date"""
    # use glob to get all files
    files = list(dir.rglob(pattern))
    return {str(f.relative_to(dir)): f.stat().st_mtime for f in files if f not in whitelist}

def _is_stale(study_dir: Path, summary_path: Path) -> bool:
    mtimes_path = study_dir / "_last_modification_times.json"
    mtimes = _get_mtimes(study_dir, whitelist=(summary_path,))
    if not mtimes_path.exists() or not summary_path.exists():
        stale = True
    else:
        mtimes_saved = json.loads(mtimes_path.read_text())
        stale = mtimes_saved != mtimes
    mtimes_path.write_text(json.dumps(mtimes))
    return stale

def get_all_task_messages(exp_dir, max_n_exp=None):
    result_list = list(yield_all_exp_results(exp_dir, progress_fn=tqdm))

    if max_n_exp is not None:
        result_list = random.sample(result_list, min(max_n_exp, len(result_list)))

    task_messages = defaultdict(list)
    for exp_result in tqdm(result_list):
        task_name = exp_result.exp_args.env_args.task_name
        for step in exp_result.steps_info:
            try:
                task_messages[task_name].append(step.task_info["message"])
            except (KeyError, TypeError):
                pass

    # count identical task messages:
    for task_name, messages in task_messages.items():
        unique_messages, count = np.unique(messages, return_counts=True)
        # sort them
        print(task_name)
        for msg, count in sorted(zip(unique_messages, count), key=lambda x: x[1], reverse=True):
            print(f"{count}x : {msg}")
        print()


    report = _extract_ablation_study(report, progression=progression)
    return report

def _get_avg_order(df: pd.DataFrame, row: pd.Series):
    """Return the average order for the given row."""
    df = df.reset_index(level=0, drop=True, inplace=False)
    # df.sort_index(inplace=True)

    sub_df = df.loc[row.name]
    orders = [get_exp_result(exp_dir).exp_args.order for exp_dir in sub_df.exp_dir]
    orders = [order for order in orders if order is not None]
    if len(orders) == 0:
        return None
    return np.mean(orders)

def _sort_order(result_df, report):
    """Add a column to the report with the average order for each agent and sort."""

    def add_order(row):
        return _get_avg_order(result_df, row)

    report["avg_order"] = report.apply(add_order, axis=1)
    return report.sort_values("avg_order", ascending=True)

def global_report(
    result_df: pd.DataFrame,
    reduce_fn=summarize,
    rename_index=lambda name: name.replace("agent.flags.", ""),
):
    """Produce a report that summarize all tasks and all episodes for each
    agent.

    Args:
        result_df: The result dataframe as returned by load_result_df.
        reduce_fn: The function to use to reduce the sub dataframe. By default
            this is summarize.
        rename_index: Function to rename the index. By default we remove the prefix
            "agent.flags."


formatter

    report.reset_index(inplace=True)

    if print_only:
        columns = [print_only] + columns
        report = report[columns]

    styled_report = set_wrap_style(report)

    display(styled_report)

def shrink_columns(df, also_wrap_index=True):
    """Make the column names more compact by replacing underscores with newlines"""
    df = df.copy()

    df.columns = [col.replace("_", "\n") for col in df.columns]
    if also_wrap_index:
        df.index.names = [name.replace("_", "\n") for name in df.index.names]

    # Define a formatter function that formats float numbers without trailing zeros
    def formatter(x):
        if isinstance(x, float):
            return "{:.10f}".format(x).rstrip("0").rstrip(".")
        return x

    return df.map(formatter)

def set_wrap_style(df):
    return df.style.set_table_styles([{"selector": "th", "props": [("white-space", "pre-wrap")]}])

# ------------
# Error Utils
# ------------

def map_err_key(err_msg: str):
    if err_msg is None:
        return err_msg

    # remove logs from the message if any
    err_msg = err_msg[: err_msg.find("=== logs ===")].rstrip()
    regex_replacements = [


            continue

        error = categorize_error(row)

        if error != current_error:
            if current_error is not None:
                print(f"{current_error.ljust(40)} : {str(error_count).rjust(5)} times")

            current_error = error
            error_count = 1
        else:
            error_count += 1

    if current_error is not None:
        print(f"{current_error.ljust(40)} : {str(error_count).rjust(5)} times")

def report_different_errors(sub_df):
    """Report the different errors in the dataframe."""

    def _categorize_error(row):
        if pd.isna(row.err_msg):
            record = {}
        else:
            record = {
                err_class: err_fn(row.err_msg, row.stack_trace)
                for err_class, err_fn in ERR_CLASS_MAP.items()
            }
            record["other_err"] = np.sum(list(record.values())) == 0
            record["any_err"] = True

        return pd.Series(record)

    error_report = sub_df.apply(_categorize_error, axis=1).sum(skipna=True)

    # TODO: fix this bug
    assert isinstance(error_report, pd.DataFrame), "Expected a DataFrame, got a Series."

    return error_report

# ===============

def _benchmark_from_task_name(task_name: str):
    """Extract the benchmark from the task name."""
    # TODO should be more robost, e.g. handle workarna.L1, workarena.L2, etc.
    return task_name.split(".")[0]

def summarize_study(result_df: pd.DataFrame) -> pd.DataFrame:
    """Create a summary of the study. Similar to global report, but handles single agent differently."""


src.agentlab.analyze.tapes

import json
import logging
import sys
from collections import defaultdict
from pathlib import Path

import numpy as np
import yaml
from tapeagents.core import Step, StepMetadata
from tapeagents.observe import retrieve_all_llm_calls
from tapeagents.renderers.camera_ready_renderer import CameraReadyRenderer
from tapeagents.tape_browser import TapeBrowser

from agentlab.agents.tapeagent.agent import ExtendedMetadata, Tape
from agentlab.benchmarks.gaia import step_error

logger = logging.getLogger(__name__)
fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

class WrapperStep(Step):
    content: dict

def pretty_yaml(data: dict | None) -> str:
    return yaml.dump(data, sort_keys=False, indent=2) if data else ""

class TapesRender(CameraReadyRenderer):

    @property
    def style(self):
        style = ""
        return super().style + style

    def render_step(self, step: WrapperStep, index: int, **kwargs):
        step_dict = step.content.copy()
        step_dict.pop("metadata", None)
        kind = step_dict.pop("kind", "Step")
        if kind == "set_next_node":
            return ""
        # remove empty keys
        step_dict = {k: v for k, v in step_dict.items() if v is not None and v != ""}
        if len(step_dict) == 1:
            content = list(step_dict.values())[0]
        elif kind == "page_observation":
            content = step_dict.get("text", pretty_yaml(step_dict))
            if len(content) > 100:
                summary = content[:100]
                content = f"{summary}---
{content}"
        elif kind == "python_code_action":
            content = step_dict.get("code", pretty_yaml(step_dict))
        elif kind == "code_execution_result":
            content = pretty_yaml(step_dict.get("result"))
        elif len(step_dict) == 1 and "content" in step_dict:
            content = step_dict["content"]
        elif len(step_dict) == 1 and "reasoning" in step_dict:
            content = step_dict["reasoning"]
        else:
            content = pretty_yaml(step_dict)

        if step_dict.get("error") or step_dict.get("result", {}).get("exit_code"):
            class_ = "error"
        elif kind.endswith("thought"):
            class_ = "thought"
            kind = kind[:-8]
        elif kind.endswith("action"):
            class_ = "action"
            kind = kind[:-7]
        else:
            class_ = "observation"
        return f"

{kind}

{content}

"

class TapesBrowser(TapeBrowser):
    def __init__(self, tapes_folder):
        super().__init__(Tape, tapes_folder, TapesRender(), ".json")

    def get_tape_files(self) -> list[str]:
        logger.info(f"Searching for tapes in {self.tapes_folder}")
        fpath = Path(self.tapes_folder)
        exps = [
            str(exp_dir.relative_to(fpath))
            for exp_dir in fpath.iterdir()
            if exp_dir.is_dir() and len(list(exp_dir.rglob("tape.json"))) > 0
        ]
        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]

    def load_llm_calls(self):
        sqlite_path = self.exp_path / "tapedata.sqlite"
        if sqlite_path.exists():
            try:
                self.llm_calls = {
                    call.prompt.id: call for call in retrieve_all_llm_calls(str(sqlite_path))
                }
                logger.info(f"Loaded {len(self.llm_calls)} LLM calls from {sqlite_path}")
            except Exception as e:
                logger.warning(f"Failed to load LLM calls from {sqlite_path}: {e}")
        else:
            logger.warning(f"{sqlite_path} not found")

    def get_context(self, tape: Tape) -> list:
        return []

    def get_tape_name(self, i: int, tape: Tape) -> str:
        errors = [
            bool(s.content.get("error", False) or s.content.get("result", {}).get("exit_code"))
            for s in tape.steps
        ]
        mark = "✅ " if tape.metadata.reward > 0 else ""
        if any(errors):
            mark = "⚠ "
        if tape.metadata.task.get("file_name"):
            mark += "📁 "
        number = tape.metadata.task.get("number", "")
        n = f"{tape.metadata.task.get('Level', '')}.{number} " if number else ""
        name = tape.steps[0].content["content"][:32] + "..."
        return f"{n}({len(tape.steps)}){mark}{name}"

    def get_exp_label(self, filename: str, tapes: list[Tape]) -> str:
        acc, n_solved = self.calculate_accuracy(tapes)
        errors = defaultdict(int)
        prompt_tokens_num = 0
        output_tokens_num = 0
        total_cost = 0.0
        visible_prompt_tokens_num = 0
        visible_output_tokens_num = 0
        visible_cost = 0.0
        no_result = 0
        actions = defaultdict(int)
        for llm_call in self.llm_calls.values():
            prompt_tokens_num += llm_call.prompt_length_tokens
            output_tokens_num += llm_call.output_length_tokens
            total_cost += llm_call.cost
        avg_steps = np.mean([len(tape) for tape in tapes])
        std_steps = np.std([len(tape) for tape in tapes])
        for tape in tapes:
            if tape.metadata.truncated:
                no_result += 1
            if tape.metadata.error:
                errors["fatal"] += 1
            last_action = None
            counted = set([])
            for step in tape:
                step_dict = step.content.copy()
                kind = step_dict.get("kind", "unknown")
                llm_call = self.llm_calls.get(step.metadata.prompt_id)
                if llm_call and step.metadata.prompt_id not in counted:
                    counted.add(step.metadata.prompt_id)
                    visible_prompt_tokens_num += llm_call.prompt_length_tokens
                    visible_output_tokens_num += llm_call.output_length_tokens
                    visible_cost += llm_call.cost
                if kind.endswith("action"):
                    actions[kind] += 1
                    last_action = kind
                if error := self.get_step_error(step_dict, last_action):
                    errors[error] += 1
        timers, timer_counts = self.aggregate_timer_times(tapes)
        html = f"
Solved {acc:.2f}%, {n_solved} out of {len(tapes)}
"
        if "all" in filename:
            html += f"Prompt tokens: {prompt_tokens_num}
Output tokens: {output_tokens_num}
Cost: {total_cost:.2f} USD
Visible
"
        html += f"Prompt tokens: {visible_prompt_tokens_num}
Output tokens: {visible_output_tokens_num}
Cost: {visible_cost:.2f} USD"
        html += f"
Steps per tape: {avg_steps:.1f} ± {std_steps:.1f}
"
        if errors:
            errors_str = "
".join(f"{k}: {v}" for k, v in errors.items())
            html += f"
No result: {no_result}
"
            html += f"
Errors: {sum(errors.values())}
{errors_str}"
        if actions:
            actions_str = "
".join(f"{k}: {v}" for k, v in actions.items())
            html += f"
Actions: {sum(actions.values())}
{actions_str}"
        if timers:
            timers_str = "
".join(
                f"{'execute ' if k.endswith('action') else ''}{k}: {v:.1f} sec, avg. {v/timer_counts[k]:.1f} sec"
                for k, v in timers.items()
            )
            html += f"
Timings
{timers_str}"
        return html

    def get_step_error(self, step_dict: dict, last_action: str | None) -> str:
        return step_error(step_dict, last_action)

    def calculate_accuracy(self, tapes: list[Tape]) -> tuple[float, int]:
        solved = [tape.metadata.reward for tape in tapes]
        accuracy = 100 * (sum(solved) / len(solved) if solved else 0.0)
        return accuracy, int(sum(solved))

    def aggregate_timer_times(self, tapes: list[Tape]):
        timer_sums = defaultdict(float)
        timer_counts = defaultdict(int)
        for tape in tapes:
            timers = tape.metadata.other.get("timers", {})
            for timer_name, exec_time in timers.items():
                timer_sums[timer_name] += exec_time
                timer_counts[timer_name] += 1
            for step in tape.steps:
                action_kind = step.metadata.other.get("action_kind")
                action_execution_time = step.metadata.other.get("action_execution_time")
                if action_kind and action_execution_time:
                    timer_sums[action_kind] += action_execution_time
                    timer_counts[action_kind] += 1
        return dict(timer_sums), dict(timer_counts)

    def load_tapes(self, exp_dir: str) -> list[Tape]:
        tapes: list[Tape] = []
        fpath = Path(self.tapes_folder) / exp_dir
        for json_file in fpath.rglob("tape.json"):
            if json_file.stat().st_size == 0:
                logger.warning(f"Empty tape file: {json_file}")
                continue
            try:
                with open(json_file) as f:
                    tape_dict = json.load(f)
                    tape = Tape(steps=[], metadata=ExtendedMetadata(**tape_dict["metadata"]))
                    tape.steps = [
                        WrapperStep(content=s, metadata=StepMetadata(**s["metadata"]))
                        for s in tape_dict["steps"]
                    ]
                    tapes.append(tape)
            except Exception as e:
                logger.warning(f"Failed to load {json_file}: {e}")
        logger.info(f"Loaded {len(tapes)} tapes from {fpath}")
        self.exp_path = fpath
        return sorted(
            tapes,
            key=lambda x: f"{x.metadata.task.get('Level', '')}{x.metadata.task.get('number', 0):03d}",
        )

    def save_annotation(self, step: int, annotation: str, tape_id: int):
        pass

if __name__ == "__main__":
    results_dir = sys.argv[1] if len(sys.argv) > 1 else "~/agentlab_results/"
    tapes_browser = TapesBrowser(Path(results_dir).expanduser())
    tapes_browser.launch()


WrapperStep

import logging
import sys
from collections import defaultdict
from pathlib import Path

import numpy as np
import yaml
from tapeagents.core import Step, StepMetadata
from tapeagents.observe import retrieve_all_llm_calls
from tapeagents.renderers.camera_ready_renderer import CameraReadyRenderer
from tapeagents.tape_browser import TapeBrowser

from agentlab.agents.tapeagent.agent import ExtendedMetadata, Tape
from agentlab.benchmarks.gaia import step_error

logger = logging.getLogger(__name__)
fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

class WrapperStep(Step):
    content: dict

def pretty_yaml(data: dict | None) -> str:
    return yaml.dump(data, sort_keys=False, indent=2) if data else ""

class TapesRender(CameraReadyRenderer):

    @property
    def style(self):
        style = "<style>.thought {{ background-color: #ffffba !important; }};</style>"
        return super().style + style

    def render_step(self, step: WrapperStep, index: int, **kwargs):
        step_dict = step.content.copy()
        step_dict.pop("metadata", None)
        kind = step_dict.pop("kind", "Step")
        if kind == "set_next_node":
            return ""
        # remove empty keys


import numpy as np
import yaml
from tapeagents.core import Step, StepMetadata
from tapeagents.observe import retrieve_all_llm_calls
from tapeagents.renderers.camera_ready_renderer import CameraReadyRenderer
from tapeagents.tape_browser import TapeBrowser

from agentlab.agents.tapeagent.agent import ExtendedMetadata, Tape
from agentlab.benchmarks.gaia import step_error

logger = logging.getLogger(__name__)
fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

class WrapperStep(Step):
    content: dict

def pretty_yaml(data: dict | None) -> str:
    return yaml.dump(data, sort_keys=False, indent=2) if data else ""

class TapesRender(CameraReadyRenderer):

    @property
    def style(self):
        style = "<style>.thought {{ background-color: #ffffba !important; }};</style>"
        return super().style + style

    def render_step(self, step: WrapperStep, index: int, **kwargs):
        step_dict = step.content.copy()
        step_dict.pop("metadata", None)
        kind = step_dict.pop("kind", "Step")
        if kind == "set_next_node":
            return ""
        # remove empty keys
        step_dict = {k: v for k, v in step_dict.items() if v is not None and v != ""}
        if len(step_dict) == 1:
            content = list(step_dict.values())[0]
        elif kind == "page_observation":


TapesRender

from tapeagents.observe import retrieve_all_llm_calls
from tapeagents.renderers.camera_ready_renderer import CameraReadyRenderer
from tapeagents.tape_browser import TapeBrowser

from agentlab.agents.tapeagent.agent import ExtendedMetadata, Tape
from agentlab.benchmarks.gaia import step_error

logger = logging.getLogger(__name__)
fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

class WrapperStep(Step):
    content: dict

def pretty_yaml(data: dict | None) -> str:
    return yaml.dump(data, sort_keys=False, indent=2) if data else ""

class TapesRender(CameraReadyRenderer):

    @property
    def style(self):
        style = ""
        return super().style + style

    def render_step(self, step: WrapperStep, index: int, **kwargs):
        step_dict = step.content.copy()
        step_dict.pop("metadata", None)
        kind = step_dict.pop("kind", "Step")
        if kind == "set_next_node":
            return ""
        # remove empty keys
        step_dict = {k: v for k, v in step_dict.items() if v is not None and v != ""}
        if len(step_dict) == 1:
            content = list(step_dict.values())[0]
        elif kind == "page_observation":
            content = step_dict.get("text", pretty_yaml(step_dict))
            if len(content) > 100:
                summary = content[:100]
                content = f"{summary}---
{content}"
        elif kind == "python_code_action":
            content = step_dict.get("code", pretty_yaml(step_dict))
        elif kind == "code_execution_result":
            content = pretty_yaml(step_dict.get("result"))
        elif len(step_dict) == 1 and "content" in step_dict:
            content = step_dict["content"]
        elif len(step_dict) == 1 and "reasoning" in step_dict:
            content = step_dict["reasoning"]
        else:
            content = pretty_yaml(step_dict)

        if step_dict.get("error") or step_dict.get("result", {}).get("exit_code"):
            class_ = "error"
        elif kind.endswith("thought"):
            class_ = "thought"
            kind = kind[:-8]
        elif kind.endswith("action"):
            class_ = "action"
            kind = kind[:-7]
        else:
            class_ = "observation"
        return f"

{kind}

{content}

"

class TapesBrowser(TapeBrowser):
    def __init__(self, tapes_folder):
        super().__init__(Tape, tapes_folder, TapesRender(), ".json")

    def get_tape_files(self) -> list[str]:
        logger.info(f"Searching for tapes in {self.tapes_folder}")
        fpath = Path(self.tapes_folder)
        exps = [
            str(exp_dir.relative_to(fpath))
            for exp_dir in fpath.iterdir()
            if exp_dir.is_dir() and len(list(exp_dir.rglob("tape.json"))) > 0
        ]
        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]


TapesBrowser

elif len(step_dict) == 1 and "content" in step_dict:
            content = step_dict["content"]
        elif len(step_dict) == 1 and "reasoning" in step_dict:
            content = step_dict["reasoning"]
        else:
            content = pretty_yaml(step_dict)

        if step_dict.get("error") or step_dict.get("result", {}).get("exit_code"):
            class_ = "error"
        elif kind.endswith("thought"):
            class_ = "thought"
            kind = kind[:-8]
        elif kind.endswith("action"):
            class_ = "action"
            kind = kind[:-7]
        else:
            class_ = "observation"
        return f"

{kind}

{content}

"

class TapesBrowser(TapeBrowser):
    def __init__(self, tapes_folder):
        super().__init__(Tape, tapes_folder, TapesRender(), ".json")

    def get_tape_files(self) -> list[str]:
        logger.info(f"Searching for tapes in {self.tapes_folder}")
        fpath = Path(self.tapes_folder)
        exps = [
            str(exp_dir.relative_to(fpath))
            for exp_dir in fpath.iterdir()
            if exp_dir.is_dir() and len(list(exp_dir.rglob("tape.json"))) > 0
        ]
        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]

    def load_llm_calls(self):
        sqlite_path = self.exp_path / "tapedata.sqlite"
        if sqlite_path.exists():
            try:
                self.llm_calls = {
                    call.prompt.id: call for call in retrieve_all_llm_calls(str(sqlite_path))
                }
                logger.info(f"Loaded {len(self.llm_calls)} LLM calls from {sqlite_path}")
            except Exception as e:
                logger.warning(f"Failed to load LLM calls from {sqlite_path}: {e}")
        else:
            logger.warning(f"{sqlite_path} not found")

    def get_context(self, tape: Tape) -> list:
        return []

    def get_tape_name(self, i: int, tape: Tape) -> str:
        errors = [
            bool(s.content.get("error", False) or s.content.get("result", {}).get("exit_code"))
            for s in tape.steps
        ]
        mark = "✅ " if tape.metadata.reward > 0 else ""
        if any(errors):
            mark = "⚠ "
        if tape.metadata.task.get("file_name"):
            mark += "📁 "
        number = tape.metadata.task.get("number", "")
        n = f"{tape.metadata.task.get('Level', '')}.{number} " if number else ""
        name = tape.steps[0].content["content"][:32] + "..."
        return f"{n}({len(tape.steps)}){mark}{name}"

    def get_exp_label(self, filename: str, tapes: list[Tape]) -> str:
        acc, n_solved = self.calculate_accuracy(tapes)
        errors = defaultdict(int)
        prompt_tokens_num = 0
        output_tokens_num = 0
        total_cost = 0.0
        visible_prompt_tokens_num = 0
        visible_output_tokens_num = 0
        visible_cost = 0.0
        no_result = 0
        actions = defaultdict(int)
        for llm_call in self.llm_calls.values():
            prompt_tokens_num += llm_call.prompt_length_tokens
            output_tokens_num += llm_call.output_length_tokens
            total_cost += llm_call.cost
        avg_steps = np.mean([len(tape) for tape in tapes])
        std_steps = np.std([len(tape) for tape in tapes])
        for tape in tapes:
            if tape.metadata.truncated:
                no_result += 1
            if tape.metadata.error:
                errors["fatal"] += 1
            last_action = None
            counted = set([])
            for step in tape:
                step_dict = step.content.copy()
                kind = step_dict.get("kind", "unknown")
                llm_call = self.llm_calls.get(step.metadata.prompt_id)
                if llm_call and step.metadata.prompt_id not in counted:
                    counted.add(step.metadata.prompt_id)
                    visible_prompt_tokens_num += llm_call.prompt_length_tokens
                    visible_output_tokens_num += llm_call.output_length_tokens
                    visible_cost += llm_call.cost
                if kind.endswith("action"):
                    actions[kind] += 1
                    last_action = kind
                if error := self.get_step_error(step_dict, last_action):
                    errors[error] += 1
        timers, timer_counts = self.aggregate_timer_times(tapes)
        html = f"
Solved {acc:.2f}%, {n_solved} out of {len(tapes)}
"
        if "all" in filename:
            html += f"Prompt tokens: {prompt_tokens_num}
Output tokens: {output_tokens_num}
Cost: {total_cost:.2f} USD
Visible
"
        html += f"Prompt tokens: {visible_prompt_tokens_num}
Output tokens: {visible_output_tokens_num}
Cost: {visible_cost:.2f} USD"
        html += f"
Steps per tape: {avg_steps:.1f} ± {std_steps:.1f}
"
        if errors:
            errors_str = "
".join(f"{k}: {v}" for k, v in errors.items())
            html += f"
No result: {no_result}
"
            html += f"
Errors: {sum(errors.values())}
{errors_str}"
        if actions:
            actions_str = "
".join(f"{k}: {v}" for k, v in actions.items())
            html += f"
Actions: {sum(actions.values())}
{actions_str}"
        if timers:
            timers_str = "
".join(
                f"{'execute ' if k.endswith('action') else ''}{k}: {v:.1f} sec, avg. {v/timer_counts[k]:.1f} sec"
                for k, v in timers.items()
            )
            html += f"
Timings
{timers_str}"
        return html

    def get_step_error(self, step_dict: dict, last_action: str | None) -> str:
        return step_error(step_dict, last_action)

    def calculate_accuracy(self, tapes: list[Tape]) -> tuple[float, int]:
        solved = [tape.metadata.reward for tape in tapes]
        accuracy = 100 * (sum(solved) / len(solved) if solved else 0.0)
        return accuracy, int(sum(solved))

    def aggregate_timer_times(self, tapes: list[Tape]):
        timer_sums = defaultdict(float)
        timer_counts = defaultdict(int)
        for tape in tapes:
            timers = tape.metadata.other.get("timers", {})
            for timer_name, exec_time in timers.items():
                timer_sums[timer_name] += exec_time
                timer_counts[timer_name] += 1
            for step in tape.steps:
                action_kind = step.metadata.other.get("action_kind")
                action_execution_time = step.metadata.other.get("action_execution_time")
                if action_kind and action_execution_time:
                    timer_sums[action_kind] += action_execution_time
                    timer_counts[action_kind] += 1
        return dict(timer_sums), dict(timer_counts)

    def load_tapes(self, exp_dir: str) -> list[Tape]:
        tapes: list[Tape] = []
        fpath = Path(self.tapes_folder) / exp_dir
        for json_file in fpath.rglob("tape.json"):
            if json_file.stat().st_size == 0:
                logger.warning(f"Empty tape file: {json_file}")
                continue
            try:
                with open(json_file) as f:
                    tape_dict = json.load(f)
                    tape = Tape(steps=[], metadata=ExtendedMetadata(**tape_dict["metadata"]))
                    tape.steps = [
                        WrapperStep(content=s, metadata=StepMetadata(**s["metadata"]))
                        for s in tape_dict["steps"]
                    ]
                    tapes.append(tape)
            except Exception as e:
                logger.warning(f"Failed to load {json_file}: {e}")
        logger.info(f"Loaded {len(tapes)} tapes from {fpath}")
        self.exp_path = fpath
        return sorted(
            tapes,
            key=lambda x: f"{x.metadata.task.get('Level', '')}{x.metadata.task.get('number', 0):03d}",
        )

    def save_annotation(self, step: int, annotation: str, tape_id: int):
        pass

if __name__ == "__main__":
    results_dir = sys.argv[1] if len(sys.argv) > 1 else "~/agentlab_results/"
    tapes_browser = TapesBrowser(Path(results_dir).expanduser())
    tapes_browser.launch()


style

from agentlab.agents.tapeagent.agent import ExtendedMetadata, Tape
from agentlab.benchmarks.gaia import step_error

logger = logging.getLogger(__name__)
fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

class WrapperStep(Step):
    content: dict

def pretty_yaml(data: dict | None) -> str:
    return yaml.dump(data, sort_keys=False, indent=2) if data else ""

class TapesRender(CameraReadyRenderer):

    @property
    def style(self):
        style = ""
        return super().style + style

    def render_step(self, step: WrapperStep, index: int, **kwargs):
        step_dict = step.content.copy()
        step_dict.pop("metadata", None)
        kind = step_dict.pop("kind", "Step")
        if kind == "set_next_node":
            return ""
        # remove empty keys
        step_dict = {k: v for k, v in step_dict.items() if v is not None and v != ""}
        if len(step_dict) == 1:
            content = list(step_dict.values())[0]
        elif kind == "page_observation":
            content = step_dict.get("text", pretty_yaml(step_dict))
            if len(content) > 100:
                summary = content[:100]
                content = f"{summary}---
{content}"
        elif kind == "python_code_action":
            content = step_dict.get("code", pretty_yaml(step_dict))
        elif kind == "code_execution_result":
            content = pretty_yaml(step_dict.get("result"))


logger = logging.getLogger(__name__)
fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])

class WrapperStep(Step):
    content: dict

def pretty_yaml(data: dict | None) -> str:
    return yaml.dump(data, sort_keys=False, indent=2) if data else ""

class TapesRender(CameraReadyRenderer):

    @property
    def style(self):
        style = ""
        return super().style + style

    def render_step(self, step: WrapperStep, index: int, **kwargs):
        step_dict = step.content.copy()
        step_dict.pop("metadata", None)
        kind = step_dict.pop("kind", "Step")
        if kind == "set_next_node":
            return ""
        # remove empty keys
        step_dict = {k: v for k, v in step_dict.items() if v is not None and v != ""}
        if len(step_dict) == 1:
            content = list(step_dict.values())[0]
        elif kind == "page_observation":
            content = step_dict.get("text", pretty_yaml(step_dict))
            if len(content) > 100:
                summary = content[:100]
                content = f"{summary}---
{content}"
        elif kind == "python_code_action":
            content = step_dict.get("code", pretty_yaml(step_dict))
        elif kind == "code_execution_result":
            content = pretty_yaml(step_dict.get("result"))
        elif len(step_dict) == 1 and "content" in step_dict:
            content = step_dict["content"]
        elif len(step_dict) == 1 and "reasoning" in step_dict:
            content = step_dict["reasoning"]
        else:
            content = pretty_yaml(step_dict)

        if step_dict.get("error") or step_dict.get("result", {}).get("exit_code"):
            class_ = "error"
        elif kind.endswith("thought"):
            class_ = "thought"
            kind = kind[:-8]
        elif kind.endswith("action"):
            class_ = "action"
            kind = kind[:-7]
        else:
            class_ = "observation"
        return f"

{kind}

{content}

"

class TapesBrowser(TapeBrowser):
    def __init__(self, tapes_folder):
        super().__init__(Tape, tapes_folder, TapesRender(), ".json")

    def get_tape_files(self) -> list[str]:
        logger.info(f"Searching for tapes in {self.tapes_folder}")
        fpath = Path(self.tapes_folder)
        exps = [
            str(exp_dir.relative_to(fpath))
            for exp_dir in fpath.iterdir()
            if exp_dir.is_dir() and len(list(exp_dir.rglob("tape.json"))) > 0
        ]
        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]


content = step_dict["content"]
        elif len(step_dict) == 1 and "reasoning" in step_dict:
            content = step_dict["reasoning"]
        else:
            content = pretty_yaml(step_dict)

        if step_dict.get("error") or step_dict.get("result", {}).get("exit_code"):
            class_ = "error"
        elif kind.endswith("thought"):
            class_ = "thought"
            kind = kind[:-8]
        elif kind.endswith("action"):
            class_ = "action"
            kind = kind[:-7]
        else:
            class_ = "observation"
        return f"

{kind}

{content}

"

class TapesBrowser(TapeBrowser):
    def __init__(self, tapes_folder):
        super().__init__(Tape, tapes_folder, TapesRender(), ".json")

    def get_tape_files(self) -> list[str]:
        logger.info(f"Searching for tapes in {self.tapes_folder}")
        fpath = Path(self.tapes_folder)
        exps = [
            str(exp_dir.relative_to(fpath))
            for exp_dir in fpath.iterdir()
            if exp_dir.is_dir() and len(list(exp_dir.rglob("tape.json"))) > 0
        ]
        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]

    def load_llm_calls(self):
        sqlite_path = self.exp_path / "tapedata.sqlite"
        if sqlite_path.exists():
            try:


else:
            content = pretty_yaml(step_dict)

        if step_dict.get("error") or step_dict.get("result", {}).get("exit_code"):
            class_ = "error"
        elif kind.endswith("thought"):
            class_ = "thought"
            kind = kind[:-8]
        elif kind.endswith("action"):
            class_ = "action"
            kind = kind[:-7]
        else:
            class_ = "observation"
        return f"

{kind}

{content}

"

class TapesBrowser(TapeBrowser):
    def __init__(self, tapes_folder):
        super().__init__(Tape, tapes_folder, TapesRender(), ".json")

    def get_tape_files(self) -> list[str]:
        logger.info(f"Searching for tapes in {self.tapes_folder}")
        fpath = Path(self.tapes_folder)
        exps = [
            str(exp_dir.relative_to(fpath))
            for exp_dir in fpath.iterdir()
            if exp_dir.is_dir() and len(list(exp_dir.rglob("tape.json"))) > 0
        ]
        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]

    def load_llm_calls(self):
        sqlite_path = self.exp_path / "tapedata.sqlite"
        if sqlite_path.exists():
            try:
                self.llm_calls = {
                    call.prompt.id: call for call in retrieve_all_llm_calls(str(sqlite_path))
                }
                logger.info(f"Loaded {len(self.llm_calls)} LLM calls from {sqlite_path}")
            except Exception as e:
                logger.warning(f"Failed to load LLM calls from {sqlite_path}: {e}")
        else:
            logger.warning(f"{sqlite_path} not found")

    def get_context(self, tape: Tape) -> list:
        return []


class_ = "observation"
        return f"

{kind}

{content}

"

class TapesBrowser(TapeBrowser):
    def __init__(self, tapes_folder):
        super().__init__(Tape, tapes_folder, TapesRender(), ".json")

    def get_tape_files(self) -> list[str]:
        logger.info(f"Searching for tapes in {self.tapes_folder}")
        fpath = Path(self.tapes_folder)
        exps = [
            str(exp_dir.relative_to(fpath))
            for exp_dir in fpath.iterdir()
            if exp_dir.is_dir() and len(list(exp_dir.rglob("tape.json"))) > 0
        ]
        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]

    def load_llm_calls(self):
        sqlite_path = self.exp_path / "tapedata.sqlite"
        if sqlite_path.exists():
            try:
                self.llm_calls = {
                    call.prompt.id: call for call in retrieve_all_llm_calls(str(sqlite_path))
                }
                logger.info(f"Loaded {len(self.llm_calls)} LLM calls from {sqlite_path}")
            except Exception as e:
                logger.warning(f"Failed to load LLM calls from {sqlite_path}: {e}")
        else:
            logger.warning(f"{sqlite_path} not found")

    def get_context(self, tape: Tape) -> list:
        return []

    def get_tape_name(self, i: int, tape: Tape) -> str:
        errors = [
            bool(s.content.get("error", False) or s.content.get("result", {}).get("exit_code"))


class TapesBrowser(TapeBrowser):
    def __init__(self, tapes_folder):
        super().__init__(Tape, tapes_folder, TapesRender(), ".json")

    def get_tape_files(self) -> list[str]:
        logger.info(f"Searching for tapes in {self.tapes_folder}")
        fpath = Path(self.tapes_folder)
        exps = [
            str(exp_dir.relative_to(fpath))
            for exp_dir in fpath.iterdir()
            if exp_dir.is_dir() and len(list(exp_dir.rglob("tape.json"))) > 0
        ]
        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]

    def load_llm_calls(self):
        sqlite_path = self.exp_path / "tapedata.sqlite"
        if sqlite_path.exists():
            try:
                self.llm_calls = {
                    call.prompt.id: call for call in retrieve_all_llm_calls(str(sqlite_path))
                }
                logger.info(f"Loaded {len(self.llm_calls)} LLM calls from {sqlite_path}")
            except Exception as e:
                logger.warning(f"Failed to load LLM calls from {sqlite_path}: {e}")
        else:
            logger.warning(f"{sqlite_path} not found")

    def get_context(self, tape: Tape) -> list:
        return []

    def get_tape_name(self, i: int, tape: Tape) -> str:
        errors = [
            bool(s.content.get("error", False) or s.content.get("result", {}).get("exit_code"))
            for s in tape.steps
        ]
        mark = "✅ " if tape.metadata.reward > 0 else ""
        if any(errors):
            mark = "⚠ "
        if tape.metadata.task.get("file_name"):
            mark += "📁 "
        number = tape.metadata.task.get("number", "")
        n = f"{tape.metadata.task.get('Level', '')}.{number} " if number else ""
        name = tape.steps[0].content["content"][:32] + "..."
        return f"{n}({len(tape.steps)}){mark}{name}"

    def get_exp_label(self, filename: str, tapes: list[Tape]) -> str:


        assert exps, f"No experiments found in {self.tapes_folder}"
        logger.info(f"Found {len(exps)} experiments in {self.tapes_folder}")
        return sorted(exps)

    def get_steps(self, tape: dict) -> list:
        return tape["steps"]

    def load_llm_calls(self):
        sqlite_path = self.exp_path / "tapedata.sqlite"
        if sqlite_path.exists():
            try:
                self.llm_calls = {
                    call.prompt.id: call for call in retrieve_all_llm_calls(str(sqlite_path))
                }
                logger.info(f"Loaded {len(self.llm_calls)} LLM calls from {sqlite_path}")
            except Exception as e:
                logger.warning(f"Failed to load LLM calls from {sqlite_path}: {e}")
        else:
            logger.warning(f"{sqlite_path} not found")

    def get_context(self, tape: Tape) -> list:
        return []

    def get_tape_name(self, i: int, tape: Tape) -> str:
        errors = [
            bool(s.content.get("error", False) or s.content.get("result", {}).get("exit_code"))
            for s in tape.steps
        ]
        mark = "✅ " if tape.metadata.reward > 0 else ""
        if any(errors):
            mark = "⚠ "
        if tape.metadata.task.get("file_name"):
            mark += "📁 "
        number = tape.metadata.task.get("number", "")
        n = f"{tape.metadata.task.get('Level', '')}.{number} " if number else ""
        name = tape.steps[0].content["content"][:32] + "..."
        return f"{n}({len(tape.steps)}){mark}{name}"

    def get_exp_label(self, filename: str, tapes: list[Tape]) -> str:
        acc, n_solved = self.calculate_accuracy(tapes)
        errors = defaultdict(int)
        prompt_tokens_num = 0


    def get_steps(self, tape: dict) -> list:
        return tape["steps"]

    def load_llm_calls(self):
        sqlite_path = self.exp_path / "tapedata.sqlite"
        if sqlite_path.exists():
            try:
                self.llm_calls = {
                    call.prompt.id: call for call in retrieve_all_llm_calls(str(sqlite_path))
                }
                logger.info(f"Loaded {len(self.llm_calls)} LLM calls from {sqlite_path}")
            except Exception as e:
                logger.warning(f"Failed to load LLM calls from {sqlite_path}: {e}")
        else:
            logger.warning(f"{sqlite_path} not found")

    def get_context(self, tape: Tape) -> list:
        return []

    def get_tape_name(self, i: int, tape: Tape) -> str:
        errors = [
            bool(s.content.get("error", False) or s.content.get("result", {}).get("exit_code"))
            for s in tape.steps
        ]
        mark = "✅ " if tape.metadata.reward > 0 else ""
        if any(errors):
            mark = "⚠ "
        if tape.metadata.task.get("file_name"):
            mark += "📁 "
        number = tape.metadata.task.get("number", "")
        n = f"{tape.metadata.task.get('Level', '')}.{number} " if number else ""
        name = tape.steps[0].content["content"][:32] + "..."
        return f"{n}({len(tape.steps)}){mark}{name}"

    def get_exp_label(self, filename: str, tapes: list[Tape]) -> str:
        acc, n_solved = self.calculate_accuracy(tapes)
        errors = defaultdict(int)
        prompt_tokens_num = 0
        output_tokens_num = 0
        total_cost = 0.0
        visible_prompt_tokens_num = 0
        visible_output_tokens_num = 0
        visible_cost = 0.0
        no_result = 0
        actions = defaultdict(int)
        for llm_call in self.llm_calls.values():
            prompt_tokens_num += llm_call.prompt_length_tokens
            output_tokens_num += llm_call.output_length_tokens
            total_cost += llm_call.cost
        avg_steps = np.mean([len(tape) for tape in tapes])
        std_steps = np.std([len(tape) for tape in tapes])
        for tape in tapes:
            if tape.metadata.truncated:


logger.warning(f"{sqlite_path} not found")

    def get_context(self, tape: Tape) -> list:
        return []

    def get_tape_name(self, i: int, tape: Tape) -> str:
        errors = [
            bool(s.content.get("error", False) or s.content.get("result", {}).get("exit_code"))
            for s in tape.steps
        ]
        mark = "✅ " if tape.metadata.reward > 0 else ""
        if any(errors):
            mark = "⚠ "
        if tape.metadata.task.get("file_name"):
            mark += "📁 "
        number = tape.metadata.task.get("number", "")
        n = f"{tape.metadata.task.get('Level', '')}.{number} " if number else ""
        name = tape.steps[0].content["content"][:32] + "..."
        return f"{n}({len(tape.steps)}){mark}{name}"

    def get_exp_label(self, filename: str, tapes: list[Tape]) -> str:
        acc, n_solved = self.calculate_accuracy(tapes)
        errors = defaultdict(int)
        prompt_tokens_num = 0
        output_tokens_num = 0
        total_cost = 0.0
        visible_prompt_tokens_num = 0
        visible_output_tokens_num = 0
        visible_cost = 0.0
        no_result = 0
        actions = defaultdict(int)
        for llm_call in self.llm_calls.values():
            prompt_tokens_num += llm_call.prompt_length_tokens
            output_tokens_num += llm_call.output_length_tokens
            total_cost += llm_call.cost
        avg_steps = np.mean([len(tape) for tape in tapes])
        std_steps = np.std([len(tape) for tape in tapes])
        for tape in tapes:
            if tape.metadata.truncated:
                no_result += 1
            if tape.metadata.error:
                errors["fatal"] += 1
            last_action = None
            counted = set([])
            for step in tape:
                step_dict = step.content.copy()
                kind = step_dict.get("kind", "unknown")
                llm_call = self.llm_calls.get(step.metadata.prompt_id)
                if llm_call and step.metadata.prompt_id not in counted:
                    counted.add(step.metadata.prompt_id)
                    visible_prompt_tokens_num += llm_call.prompt_length_tokens
                    visible_output_tokens_num += llm_call.output_length_tokens
                    visible_cost += llm_call.cost
                if kind.endswith("action"):
                    actions[kind] += 1
                    last_action = kind
                if error := self.get_step_error(step_dict, last_action):
                    errors[error] += 1
        timers, timer_counts = self.aggregate_timer_times(tapes)
        html = f"
Solved {acc:.2f}%, {n_solved} out of {len(tapes)}
"
        if "all" in filename:
            html += f"Prompt tokens: {prompt_tokens_num}
Output tokens: {output_tokens_num}
Cost: {total_cost:.2f} USD
Visible
"
        html += f"Prompt tokens: {visible_prompt_tokens_num}
Output tokens: {visible_output_tokens_num}
Cost: {visible_cost:.2f} USD"
        html += f"
Steps per tape: {avg_steps:.1f} ± {std_steps:.1f}
"
        if errors:
            errors_str = "
".join(f"{k}: {v}" for k, v in errors.items())
            html += f"
No result: {no_result}
"
            html += f"
Errors: {sum(errors.values())}
{errors_str}"
        if actions:
            actions_str = "
".join(f"{k}: {v}" for k, v in actions.items())
            html += f"
Actions: {sum(actions.values())}
{actions_str}"
        if timers:
            timers_str = "
".join(
                f"{'execute ' if k.endswith('action') else ''}{k}: {v:.1f} sec, avg. {v/timer_counts[k]:.1f} sec"
                for k, v in timers.items()
            )
            html += f"
Timings
{timers_str}"
        return html

    def get_step_error(self, step_dict: dict, last_action: str | None) -> str:
        return step_error(step_dict, last_action)

    def calculate_accuracy(self, tapes: list[Tape]) -> tuple[float, int]:
        solved = [tape.metadata.reward for tape in tapes]
        accuracy = 100 * (sum(solved) / len(solved) if solved else 0.0)
        return accuracy, int(sum(solved))

    def aggregate_timer_times(self, tapes: list[Tape]):
        timer_sums = defaultdict(float)
        timer_counts = defaultdict(int)
        for tape in tapes:
            timers = tape.metadata.other.get("timers", {})
            for timer_name, exec_time in timers.items():
                timer_sums[timer_name] += exec_time
                timer_counts[timer_name] += 1
            for step in tape.steps:
                action_kind = step.metadata.other.get("action_kind")
                action_execution_time = step.metadata.other.get("action_execution_time")


html = f"
Solved {acc:.2f}%, {n_solved} out of {len(tapes)}
"
        if "all" in filename:
            html += f"Prompt tokens: {prompt_tokens_num}
Output tokens: {output_tokens_num}
Cost: {total_cost:.2f} USD
Visible
"
        html += f"Prompt tokens: {visible_prompt_tokens_num}
Output tokens: {visible_output_tokens_num}
Cost: {visible_cost:.2f} USD"
        html += f"
Steps per tape: {avg_steps:.1f} ± {std_steps:.1f}
"
        if errors:
            errors_str = "
".join(f"{k}: {v}" for k, v in errors.items())
            html += f"
No result: {no_result}
"
            html += f"
Errors: {sum(errors.values())}
{errors_str}"
        if actions:
            actions_str = "
".join(f"{k}: {v}" for k, v in actions.items())
            html += f"
Actions: {sum(actions.values())}
{actions_str}"
        if timers:
            timers_str = "
".join(
                f"{'execute ' if k.endswith('action') else ''}{k}: {v:.1f} sec, avg. {v/timer_counts[k]:.1f} sec"
                for k, v in timers.items()
            )
            html += f"
Timings
{timers_str}"
        return html

    def get_step_error(self, step_dict: dict, last_action: str | None) -> str:
        return step_error(step_dict, last_action)

    def calculate_accuracy(self, tapes: list[Tape]) -> tuple[float, int]:
        solved = [tape.metadata.reward for tape in tapes]
        accuracy = 100 * (sum(solved) / len(solved) if solved else 0.0)
        return accuracy, int(sum(solved))

    def aggregate_timer_times(self, tapes: list[Tape]):
        timer_sums = defaultdict(float)
        timer_counts = defaultdict(int)
        for tape in tapes:
            timers = tape.metadata.other.get("timers", {})
            for timer_name, exec_time in timers.items():
                timer_sums[timer_name] += exec_time
                timer_counts[timer_name] += 1
            for step in tape.steps:
                action_kind = step.metadata.other.get("action_kind")
                action_execution_time = step.metadata.other.get("action_execution_time")
                if action_kind and action_execution_time:
                    timer_sums[action_kind] += action_execution_time
                    timer_counts[action_kind] += 1


html += f"Prompt tokens: {visible_prompt_tokens_num}
Output tokens: {visible_output_tokens_num}
Cost: {visible_cost:.2f} USD"
        html += f"
Steps per tape: {avg_steps:.1f} ± {std_steps:.1f}
"
        if errors:
            errors_str = "
".join(f"{k}: {v}" for k, v in errors.items())
            html += f"
No result: {no_result}
"
            html += f"
Errors: {sum(errors.values())}
{errors_str}"
        if actions:
            actions_str = "
".join(f"{k}: {v}" for k, v in actions.items())
            html += f"
Actions: {sum(actions.values())}
{actions_str}"
        if timers:
            timers_str = "
".join(
                f"{'execute ' if k.endswith('action') else ''}{k}: {v:.1f} sec, avg. {v/timer_counts[k]:.1f} sec"
                for k, v in timers.items()
            )
            html += f"
Timings
{timers_str}"
        return html

    def get_step_error(self, step_dict: dict, last_action: str | None) -> str:
        return step_error(step_dict, last_action)

    def calculate_accuracy(self, tapes: list[Tape]) -> tuple[float, int]:
        solved = [tape.metadata.reward for tape in tapes]
        accuracy = 100 * (sum(solved) / len(solved) if solved else 0.0)
        return accuracy, int(sum(solved))

    def aggregate_timer_times(self, tapes: list[Tape]):
        timer_sums = defaultdict(float)
        timer_counts = defaultdict(int)
        for tape in tapes:
            timers = tape.metadata.other.get("timers", {})
            for timer_name, exec_time in timers.items():
                timer_sums[timer_name] += exec_time
                timer_counts[timer_name] += 1
            for step in tape.steps:
                action_kind = step.metadata.other.get("action_kind")
                action_execution_time = step.metadata.other.get("action_execution_time")
                if action_kind and action_execution_time:
                    timer_sums[action_kind] += action_execution_time
                    timer_counts[action_kind] += 1
        return dict(timer_sums), dict(timer_counts)

    def load_tapes(self, exp_dir: str) -> list[Tape]:
        tapes: list[Tape] = []
        fpath = Path(self.tapes_folder) / exp_dir


html += f"
Errors: {sum(errors.values())}
{errors_str}"
        if actions:
            actions_str = "
".join(f"{k}: {v}" for k, v in actions.items())
            html += f"
Actions: {sum(actions.values())}
{actions_str}"
        if timers:
            timers_str = "
".join(
                f"{'execute ' if k.endswith('action') else ''}{k}: {v:.1f} sec, avg. {v/timer_counts[k]:.1f} sec"
                for k, v in timers.items()
            )
            html += f"
Timings
{timers_str}"
        return html

    def get_step_error(self, step_dict: dict, last_action: str | None) -> str:
        return step_error(step_dict, last_action)

    def calculate_accuracy(self, tapes: list[Tape]) -> tuple[float, int]:
        solved = [tape.metadata.reward for tape in tapes]
        accuracy = 100 * (sum(solved) / len(solved) if solved else 0.0)
        return accuracy, int(sum(solved))

    def aggregate_timer_times(self, tapes: list[Tape]):
        timer_sums = defaultdict(float)
        timer_counts = defaultdict(int)
        for tape in tapes:
            timers = tape.metadata.other.get("timers", {})
            for timer_name, exec_time in timers.items():
                timer_sums[timer_name] += exec_time
                timer_counts[timer_name] += 1
            for step in tape.steps:
                action_kind = step.metadata.other.get("action_kind")
                action_execution_time = step.metadata.other.get("action_execution_time")
                if action_kind and action_execution_time:
                    timer_sums[action_kind] += action_execution_time
                    timer_counts[action_kind] += 1
        return dict(timer_sums), dict(timer_counts)

    def load_tapes(self, exp_dir: str) -> list[Tape]:
        tapes: list[Tape] = []
        fpath = Path(self.tapes_folder) / exp_dir
        for json_file in fpath.rglob("tape.json"):
            if json_file.stat().st_size == 0:
                logger.warning(f"Empty tape file: {json_file}")
                continue
            try:
                with open(json_file) as f:
                    tape_dict = json.load(f)
                    tape = Tape(steps=[], metadata=ExtendedMetadata(**tape_dict["metadata"]))
                    tape.steps = [
                        WrapperStep(content=s, metadata=StepMetadata(**s["metadata"]))
                        for s in tape_dict["steps"]
                    ]
                    tapes.append(tape)
            except Exception as e:
                logger.warning(f"Failed to load {json_file}: {e}")
        logger.info(f"Loaded {len(tapes)} tapes from {fpath}")


        solved = [tape.metadata.reward for tape in tapes]
        accuracy = 100 * (sum(solved) / len(solved) if solved else 0.0)
        return accuracy, int(sum(solved))

    def aggregate_timer_times(self, tapes: list[Tape]):
        timer_sums = defaultdict(float)
        timer_counts = defaultdict(int)
        for tape in tapes:
            timers = tape.metadata.other.get("timers", {})
            for timer_name, exec_time in timers.items():
                timer_sums[timer_name] += exec_time
                timer_counts[timer_name] += 1
            for step in tape.steps:
                action_kind = step.metadata.other.get("action_kind")
                action_execution_time = step.metadata.other.get("action_execution_time")
                if action_kind and action_execution_time:
                    timer_sums[action_kind] += action_execution_time
                    timer_counts[action_kind] += 1
        return dict(timer_sums), dict(timer_counts)

    def load_tapes(self, exp_dir: str) -> list[Tape]:
        tapes: list[Tape] = []
        fpath = Path(self.tapes_folder) / exp_dir
        for json_file in fpath.rglob("tape.json"):
            if json_file.stat().st_size == 0:
                logger.warning(f"Empty tape file: {json_file}")
                continue
            try:
                with open(json_file) as f:
                    tape_dict = json.load(f)
                    tape = Tape(steps=[], metadata=ExtendedMetadata(**tape_dict["metadata"]))
                    tape.steps = [
                        WrapperStep(content=s, metadata=StepMetadata(**s["metadata"]))
                        for s in tape_dict["steps"]
                    ]
                    tapes.append(tape)
            except Exception as e:
                logger.warning(f"Failed to load {json_file}: {e}")
        logger.info(f"Loaded {len(tapes)} tapes from {fpath}")
        self.exp_path = fpath
        return sorted(
            tapes,
            key=lambda x: f"{x.metadata.task.get('Level', '')}{x.metadata.task.get('number', 0):03d}",
        )

    def save_annotation(self, step: int, annotation: str, tape_id: int):
        pass

if __name__ == "__main__":
    results_dir = sys.argv[1] if len(sys.argv) > 1 else "~/agentlab_results/"
    tapes_browser = TapesBrowser(Path(results_dir).expanduser())
    tapes_browser.launch()