#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Parse ICBCBench LaTeX result tables and generate data/leaderboard.csv. Reads: - main_results.tex EN/ZH Objective/Subjective/Overall - results_subset_en.tex EN Expert/Citation/Source details - results_subset_zh.tex ZH Expert/Citation/Source details - objective_public_CalibErr.tex All-language Accuracy / Calibration Error """ from __future__ import annotations import re import csv from pathlib import Path from collections import defaultdict PROJECT_ROOT = Path(__file__).resolve().parent.parent def parse_main_results(tex_file: Path) -> dict[str, dict]: """Parse main_results.tex and return {model: {objective_en, subjective_en, overall_en, objective_zh, subjective_zh, overall_zh}}.""" text = tex_file.read_text(encoding="utf-8") rows = {} for line in text.splitlines(): line = line.strip() if not line or line.startswith("\\") or line.startswith("%"): continue # Match lines like: # Gemini-deep-research & 50.00 & 64.77 & \underline{57.38} & 52.50 & \textbf{65.69} & \underline{59.09} \\ parts = [p.strip() for p in line.split("&")] if len(parts) != 7: continue model = parts[0].strip() values = [] for p in parts[1:]: # Strip LaTeX formatting commands but keep their arguments val = re.sub(r"\\(textbf|underline|rowcolor)\{([^}]*)\}", r"\2", p) # Drop structural LaTeX commands and trailing \ val = re.sub(r"\\(multicolumn|cmidrule|toprule|midrule|bottomrule).*", "", val) val = val.replace("\\", "").replace("{", "").replace("}", "").strip() if val in ("", "--"): values.append(None) else: try: values.append(float(val)) except ValueError: values.append(None) if all(v is not None for v in values): rows[model] = { "objective_en": values[0], "subjective_en": values[1], "overall_en": values[2], "objective_zh": values[3], "subjective_zh": values[4], "overall_zh": values[5], } return rows def parse_subset(tex_file: Path) -> dict[str, dict]: """Parse results_subset_en.tex or results_subset_zh.tex. Returns {model: {objective_text, objective_all, expert, citation, source, overall}}. """ text = tex_file.read_text(encoding="utf-8") rows = {} for line in text.splitlines(): line = line.strip() if not line or line.startswith("\\") or line.startswith("%"): continue parts = [p.strip() for p in line.split("&")] if len(parts) != 7: continue model = parts[0].strip() values = [] for p in parts[1:]: val = re.sub(r"\\(textbf|underline|rowcolor)\{([^}]*)\}", r"\2", p) val = re.sub(r"\\(multicolumn|cmidrule|toprule|midrule|bottomrule).*", "", val) val = val.replace("\\", "").replace("{", "").replace("}", "").strip() if val in ("", "--"): values.append(None) else: try: values.append(float(val)) except ValueError: values.append(None) if values[-1] is not None: # overall is required rows[model] = { "objective_text": values[0], "objective_all": values[1], "expert": values[2], "citation": values[3], "source": values[4], "overall": values[5], } return rows def parse_calibration(tex_file: Path) -> dict[str, dict]: """Parse objective_public_CalibErr.tex. Returns {model: {accuracy, calibration_error}}.""" text = tex_file.read_text(encoding="utf-8") rows = {} for line in text.splitlines(): line = line.strip() if not line or line.startswith("\\") or line.startswith("%"): continue parts = [p.strip() for p in line.split("&")] if len(parts) != 3: continue model = parts[0].strip() values = [] for p in parts[1:]: val = p.replace("\\", "").strip() if val in ("", "--"): values.append(None) else: try: values.append(float(val)) except ValueError: values.append(None) if all(v is not None for v in values): rows[model] = {"accuracy": values[0], "calibration_error": values[1]} return rows def merge_scores( main: dict[str, dict], en_detail: dict[str, dict], zh_detail: dict[str, dict], calib: dict[str, dict], ) -> list[dict]: models = sorted(main.keys()) results = [] for model in models: m = main[model] en = en_detail.get(model, {}) zh = zh_detail.get(model, {}) cal = calib.get(model, {}) # Aggregate citation / source from available detailed tables. # Prefer EN values when both exist, otherwise use whichever is available. citation = en.get("citation") if en.get("citation") is not None else zh.get("citation") source = en.get("source") if en.get("source") is not None else zh.get("source") # Expert score: average of EN and ZH expert if both exist. experts = [v for v in [en.get("expert"), zh.get("expert")] if v is not None] expert_avg = sum(experts) / len(experts) if experts else None objective_avg = (m["objective_en"] + m["objective_zh"]) / 2 subjective_avg = (m["subjective_en"] + m["subjective_zh"]) / 2 overall = (m["overall_en"] + m["overall_zh"]) / 2 results.append({ "model": model, "overall": overall, "objective_en": m["objective_en"], "objective_zh": m["objective_zh"], "objective_avg": objective_avg, "subjective_en": m["subjective_en"], "subjective_zh": m["subjective_zh"], "subjective_avg": subjective_avg, "expert_avg": expert_avg, "citation_score": citation, "source_quality": source, "rmsce": cal.get("calibration_error"), }) # Sort by overall descending, then objective_avg, then subjective_avg. results.sort( key=lambda x: (x["overall"], x["objective_avg"], x["subjective_avg"]), reverse=True, ) return results def write_leaderboard(results: list[dict], output_file: Path): fieldnames = [ "model", "overall", "objective_en", "objective_zh", "objective_avg", "subjective_en", "subjective_zh", "subjective_avg", "expert_avg", "citation_score", "source_quality", "rmsce", ] with open(output_file, "w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for r in results: row = {k: r[k] for k in fieldnames} for k in fieldnames: if k == "model": continue val = row[k] row[k] = f"{val:.2f}" if val is not None else "-" writer.writerow(row) print(f"Wrote {len(results)} models to {output_file}") def main(): main_file = PROJECT_ROOT / "main_results.tex" en_file = PROJECT_ROOT / "results_subset_en.tex" zh_file = PROJECT_ROOT / "results_subset_zh.tex" calib_file = PROJECT_ROOT / "objective_public_CalibErr.tex" output_file = PROJECT_ROOT / "data" / "leaderboard.csv" main = parse_main_results(main_file) en_detail = parse_subset(en_file) zh_detail = parse_subset(zh_file) calib = parse_calibration(calib_file) print(f"Parsed {len(main)} models from main results") print(f"Parsed {len(en_detail)} EN detailed rows") print(f"Parsed {len(zh_detail)} ZH detailed rows") print(f"Parsed {len(calib)} calibration rows") results = merge_scores(main, en_detail, zh_detail, calib) write_leaderboard(results, output_file) if __name__ == "__main__": main()