| """Build the public leaderboard dataset from versioned raw result summaries.""" |
|
|
| from __future__ import annotations |
|
|
| import json |
| import re |
| from pathlib import Path |
|
|
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| RAW_ROOT = ROOT / "data" / "raw" |
| RAW_DIR = RAW_ROOT / "2026-08-04-expanded-benchmark" |
| V4_PRO_DIR = RAW_ROOT / "deepseek-deepseek-v4-pro_5datasets_full_20260811" |
| OUTPUT_PATH = ROOT / "data" / "results.json" |
|
|
| IDENTITY_FIELDS = ("agent_name", "agent_type", "base_model") |
| REQUIRED_METRICS = ("T1_acc", "T2_acc", "T3_acc", "T4_acc") |
| METRIC_PATTERN = re.compile( |
| r"^(?:" |
| r"T[1-4]_acc|" |
| r"T[24]_(?:MAE|sMAPE)|" |
| r"T[24]_OW_(?:sMAPE|RMSSE)_MIMIC|" |
| r"(?:FreshRetailNet|PSML|CausalChambers|MIMIC|M5)_T[1-4]_acc|" |
| r"(?:FreshRetailNet|PSML|CausalChambers|MIMIC|M5)_T[24]_" |
| r"(?:MAE|sMAPE|OW_sMAPE|OW_RMSSE)" |
| r")$" |
| ) |
|
|
|
|
| def load_records(path: Path) -> list[dict]: |
| with path.open() as file: |
| data = json.load(file) |
| if not isinstance(data, list): |
| raise ValueError(f"Expected a JSON list in {path}") |
| return data |
|
|
|
|
| def normalize_agent_type(value: str) -> str: |
| if value == "general multi-agent framework": |
| return "general agent" |
| return value |
|
|
|
|
| def normalize_agent_name(value: str) -> str: |
| if value == "TimeSeriesScientist": |
| return "TimeSeries Scientist" |
| return value |
|
|
|
|
| def normalize_record(record: dict) -> dict: |
| normalized = { |
| "agent_name": normalize_agent_name(record["agent_name"]), |
| "agent_type": normalize_agent_type(record["agent_type"]), |
| "base_model": record["base_model"], |
| } |
|
|
| for key, value in record.items(): |
| if METRIC_PATTERN.fullmatch(key): |
| normalized[key] = value |
|
|
| for metric in REQUIRED_METRICS: |
| normalized.setdefault(metric, None) |
| return normalized |
|
|
|
|
| def main() -> None: |
| sources = [ |
| RAW_DIR / "results_before_expansion.json", |
| RAW_DIR / "eval_results_summary_deepseek-deepseek-chat_M5-MIMIC-PSML-CausalChambers-FreshRetailNet_high.json", |
| RAW_DIR / "old_baselines_on_M5.json", |
| RAW_DIR / "paper_deepseek_chat_baselines.json", |
| RAW_DIR / "deepseek-v4-pro_full_r100_20260721-173343" / "summary.json", |
| V4_PRO_DIR / "eval_results_summary_deepseek-deepseek-v4-pro_5datasets_full_high.json", |
| ] |
|
|
| records_by_identity = {} |
| for source in sources: |
| for raw_record in load_records(source): |
| record = normalize_record(raw_record) |
| identity = tuple(record[field] for field in IDENTITY_FIELDS) |
| existing = records_by_identity.get(identity) |
| if existing is None: |
| records_by_identity[identity] = record |
| continue |
|
|
| for key, value in record.items(): |
| if key in IDENTITY_FIELDS or value is None: |
| continue |
| existing_value = existing.get(key) |
| if existing_value is not None and existing_value != value: |
| raise ValueError( |
| f"Conflicting metric '{key}' for configuration: {identity}" |
| ) |
| existing[key] = value |
|
|
| records = list(records_by_identity.values()) |
| records.sort(key=lambda record: (record["agent_name"].lower(), record["base_model"].lower())) |
| with OUTPUT_PATH.open("w") as file: |
| json.dump(records, file, indent=2) |
| file.write("\n") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
