utils/parse_icbcbench_tex.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Parse ICBCBench LaTeX result tables and generate data/leaderboard.csv.

Reads:
  - main_results.tex            EN/ZH Objective/Subjective/Overall
  - results_subset_en.tex       EN Expert/Citation/Source details
  - results_subset_zh.tex       ZH Expert/Citation/Source details
  - objective_public_CalibErr.tex  All-language Accuracy / Calibration Error
"""

from __future__ import annotations
import re
import csv
from pathlib import Path
from collections import defaultdict

PROJECT_ROOT = Path(__file__).resolve().parent.parent


def parse_main_results(tex_file: Path) -> dict[str, dict]:
    """Parse main_results.tex and return {model: {objective_en, subjective_en, overall_en, objective_zh, subjective_zh, overall_zh}}."""
    text = tex_file.read_text(encoding="utf-8")
    rows = {}
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith("\\") or line.startswith("%"):
            continue
        # Match lines like:
        # Gemini-deep-research                     &           50.00 &           64.77 & \underline{57.38} &           52.50 &  \textbf{65.69} & \underline{59.09} \\
        parts = [p.strip() for p in line.split("&")]
        if len(parts) != 7:
            continue
        model = parts[0].strip()
        values = []
        for p in parts[1:]:
            # Strip LaTeX formatting commands but keep their arguments
            val = re.sub(r"\\(textbf|underline|rowcolor)\{([^}]*)\}", r"\2", p)
            # Drop structural LaTeX commands and trailing \
            val = re.sub(r"\\(multicolumn|cmidrule|toprule|midrule|bottomrule).*", "", val)
            val = val.replace("\\", "").replace("{", "").replace("}", "").strip()
            if val in ("", "--"):
                values.append(None)
            else:
                try:
                    values.append(float(val))
                except ValueError:
                    values.append(None)
        if all(v is not None for v in values):
            rows[model] = {
                "objective_en": values[0],
                "subjective_en": values[1],
                "overall_en": values[2],
                "objective_zh": values[3],
                "subjective_zh": values[4],
                "overall_zh": values[5],
            }
    return rows


def parse_subset(tex_file: Path) -> dict[str, dict]:
    """Parse results_subset_en.tex or results_subset_zh.tex.
    Returns {model: {objective_text, objective_all, expert, citation, source, overall}}.
    """
    text = tex_file.read_text(encoding="utf-8")
    rows = {}
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith("\\") or line.startswith("%"):
            continue
        parts = [p.strip() for p in line.split("&")]
        if len(parts) != 7:
            continue
        model = parts[0].strip()
        values = []
        for p in parts[1:]:
            val = re.sub(r"\\(textbf|underline|rowcolor)\{([^}]*)\}", r"\2", p)
            val = re.sub(r"\\(multicolumn|cmidrule|toprule|midrule|bottomrule).*", "", val)
            val = val.replace("\\", "").replace("{", "").replace("}", "").strip()
            if val in ("", "--"):
                values.append(None)
            else:
                try:
                    values.append(float(val))
                except ValueError:
                    values.append(None)
        if values[-1] is not None:  # overall is required
            rows[model] = {
                "objective_text": values[0],
                "objective_all": values[1],
                "expert": values[2],
                "citation": values[3],
                "source": values[4],
                "overall": values[5],
            }
    return rows


def parse_calibration(tex_file: Path) -> dict[str, dict]:
    """Parse objective_public_CalibErr.tex. Returns {model: {accuracy, calibration_error}}."""
    text = tex_file.read_text(encoding="utf-8")
    rows = {}
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith("\\") or line.startswith("%"):
            continue
        parts = [p.strip() for p in line.split("&")]
        if len(parts) != 3:
            continue
        model = parts[0].strip()
        values = []
        for p in parts[1:]:
            val = p.replace("\\", "").strip()
            if val in ("", "--"):
                values.append(None)
            else:
                try:
                    values.append(float(val))
                except ValueError:
                    values.append(None)
        if all(v is not None for v in values):
            rows[model] = {"accuracy": values[0], "calibration_error": values[1]}
    return rows


def merge_scores(
    main: dict[str, dict],
    en_detail: dict[str, dict],
    zh_detail: dict[str, dict],
    calib: dict[str, dict],
) -> list[dict]:
    models = sorted(main.keys())
    results = []
    for model in models:
        m = main[model]
        en = en_detail.get(model, {})
        zh = zh_detail.get(model, {})
        cal = calib.get(model, {})

        # Aggregate citation / source from available detailed tables.
        # Prefer EN values when both exist, otherwise use whichever is available.
        citation = en.get("citation") if en.get("citation") is not None else zh.get("citation")
        source = en.get("source") if en.get("source") is not None else zh.get("source")

        # Expert score: average of EN and ZH expert if both exist.
        experts = [v for v in [en.get("expert"), zh.get("expert")] if v is not None]
        expert_avg = sum(experts) / len(experts) if experts else None

        objective_avg = (m["objective_en"] + m["objective_zh"]) / 2
        subjective_avg = (m["subjective_en"] + m["subjective_zh"]) / 2
        overall = (m["overall_en"] + m["overall_zh"]) / 2

        results.append({
            "model": model,
            "overall": overall,
            "objective_en": m["objective_en"],
            "objective_zh": m["objective_zh"],
            "objective_avg": objective_avg,
            "subjective_en": m["subjective_en"],
            "subjective_zh": m["subjective_zh"],
            "subjective_avg": subjective_avg,
            "expert_avg": expert_avg,
            "citation_score": citation,
            "source_quality": source,
            "rmsce": cal.get("calibration_error"),
        })

    # Sort by overall descending, then objective_avg, then subjective_avg.
    results.sort(
        key=lambda x: (x["overall"], x["objective_avg"], x["subjective_avg"]),
        reverse=True,
    )
    return results


def write_leaderboard(results: list[dict], output_file: Path):
    fieldnames = [
        "model", "overall", "objective_en", "objective_zh", "objective_avg",
        "subjective_en", "subjective_zh", "subjective_avg",
        "expert_avg", "citation_score", "source_quality", "rmsce",
    ]
    with open(output_file, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        for r in results:
            row = {k: r[k] for k in fieldnames}
            for k in fieldnames:
                if k == "model":
                    continue
                val = row[k]
                row[k] = f"{val:.2f}" if val is not None else "-"
            writer.writerow(row)
    print(f"Wrote {len(results)} models to {output_file}")


def main():
    main_file = PROJECT_ROOT / "main_results.tex"
    en_file = PROJECT_ROOT / "results_subset_en.tex"
    zh_file = PROJECT_ROOT / "results_subset_zh.tex"
    calib_file = PROJECT_ROOT / "objective_public_CalibErr.tex"
    output_file = PROJECT_ROOT / "data" / "leaderboard.csv"

    main = parse_main_results(main_file)
    en_detail = parse_subset(en_file)
    zh_detail = parse_subset(zh_file)
    calib = parse_calibration(calib_file)

    print(f"Parsed {len(main)} models from main results")
    print(f"Parsed {len(en_detail)} EN detailed rows")
    print(f"Parsed {len(zh_detail)} ZH detailed rows")
    print(f"Parsed {len(calib)} calibration rows")

    results = merge_scores(main, en_detail, zh_detail, calib)
    write_leaderboard(results, output_file)


if __name__ == "__main__":
    main()