File size: 3,443 Bytes
af980b2 72dff92 af980b2 d5a88f8 db3e9da 72dff92 af980b2 d5a88f8 af980b2 d5a88f8 af980b2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """Build the public leaderboard dataset from versioned raw result summaries."""
from __future__ import annotations
import json
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
RAW_ROOT = ROOT / "data" / "raw"
RAW_DIR = RAW_ROOT / "2026-08-04-expanded-benchmark"
V4_PRO_DIR = RAW_ROOT / "deepseek-deepseek-v4-pro_5datasets_full_20260811"
OUTPUT_PATH = ROOT / "data" / "results.json"
IDENTITY_FIELDS = ("agent_name", "agent_type", "base_model")
REQUIRED_METRICS = ("T1_acc", "T2_acc", "T3_acc", "T4_acc")
METRIC_PATTERN = re.compile(
r"^(?:"
r"T[1-4]_acc|"
r"T[24]_(?:MAE|sMAPE)|"
r"T[24]_OW_(?:sMAPE|RMSSE)_MIMIC|"
r"(?:FreshRetailNet|PSML|CausalChambers|MIMIC|M5)_T[1-4]_acc|"
r"(?:FreshRetailNet|PSML|CausalChambers|MIMIC|M5)_T[24]_"
r"(?:MAE|sMAPE|OW_sMAPE|OW_RMSSE)"
r")$"
)
def load_records(path: Path) -> list[dict]:
with path.open() as file:
data = json.load(file)
if not isinstance(data, list):
raise ValueError(f"Expected a JSON list in {path}")
return data
def normalize_agent_type(value: str) -> str:
if value == "general multi-agent framework":
return "general agent"
return value
def normalize_agent_name(value: str) -> str:
if value == "TimeSeriesScientist":
return "TimeSeries Scientist"
return value
def normalize_record(record: dict) -> dict:
normalized = {
"agent_name": normalize_agent_name(record["agent_name"]),
"agent_type": normalize_agent_type(record["agent_type"]),
"base_model": record["base_model"],
}
for key, value in record.items():
if METRIC_PATTERN.fullmatch(key):
normalized[key] = value
for metric in REQUIRED_METRICS:
normalized.setdefault(metric, None)
return normalized
def main() -> None:
sources = [
RAW_DIR / "results_before_expansion.json",
RAW_DIR / "eval_results_summary_deepseek-deepseek-chat_M5-MIMIC-PSML-CausalChambers-FreshRetailNet_high.json",
RAW_DIR / "old_baselines_on_M5.json",
RAW_DIR / "paper_deepseek_chat_baselines.json",
RAW_DIR / "deepseek-v4-pro_full_r100_20260721-173343" / "summary.json",
V4_PRO_DIR / "eval_results_summary_deepseek-deepseek-v4-pro_5datasets_full_high.json",
]
records_by_identity = {}
for source in sources:
for raw_record in load_records(source):
record = normalize_record(raw_record)
identity = tuple(record[field] for field in IDENTITY_FIELDS)
existing = records_by_identity.get(identity)
if existing is None:
records_by_identity[identity] = record
continue
for key, value in record.items():
if key in IDENTITY_FIELDS or value is None:
continue
existing_value = existing.get(key)
if existing_value is not None and existing_value != value:
raise ValueError(
f"Conflicting metric '{key}' for configuration: {identity}"
)
existing[key] = value
records = list(records_by_identity.values())
records.sort(key=lambda record: (record["agent_name"].lower(), record["base_model"].lower()))
with OUTPUT_PATH.open("w") as file:
json.dump(records, file, indent=2)
file.write("\n")
if __name__ == "__main__":
main()
|