GIFT-Eval style absolute metrics, Overall/By Domain tabs

src/utils.py

@@ -1,4 +1,3 @@
-import json
 import os
 import re
 from pathlib import Path
@@ -56,6 +55,8 @@ LEGACY_MODEL_NAMES = {
 
 def format_number(value):
     if isinstance(value, (int, float)):
+        if pd.isna(value):
+            return value
         if abs(value) >= 10**2:
             return f"{value:.1e}"
         return f"{value:.3f}"
@@ -78,40 +79,6 @@ def unify_freq(df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
-def load_baseline_name(root_dir: str = "results") -> str:
-    status_path = Path(root_dir) / "online_status.json"
-    if status_path.exists():
-        baseline = json.loads(status_path.read_text()).get("baseline_model")
-        if baseline:
-            return baseline
-    return "Chronos-Bolt-Base"
-
-
-def standardize_df(df: pd.DataFrame, root_dir: str = "results") -> pd.DataFrame:
-    df = df.copy()
-    df["model"] = df["model"].replace(LEGACY_MODEL_NAMES)
-
-    for metric in METRIC_COLUMNS:
-        df[metric] = pd.to_numeric(df[metric], errors="coerce")
-        df[metric] = df[metric].fillna(df[metric].mean())
-
-    baseline_name = load_baseline_name(root_dir)
-    if baseline_name not in set(df["model"]):
-        return df
-
-    group_keys = ["dataset", "term_length", "frequency"]
-    for _, group in df.groupby(group_keys):
-        baseline = group[group["model"] == baseline_name]
-        if baseline.empty:
-            continue
-        idx = baseline.index[0]
-        for metric in METRIC_COLUMNS:
-            denom = baseline.at[idx, metric]
-            if denom and denom != 0:
-                df.loc[group.index, metric] = df.loc[group.index, metric] / denom
-    return df
-
-
 def load_all_results(root_dir: str = "results") -> pd.DataFrame:
     frames = []
     for subdir, _, files in os.walk(root_dir):
@@ -143,6 +110,23 @@ def enrich_with_properties(df: pd.DataFrame, ds_properties_path: str) -> pd.Data
     return unify_freq(df)
 
 
+def prepare_results_df(
+    root_dir: str = "results",
+    ds_properties: str | None = None,
+) -> pd.DataFrame:
+    if ds_properties is None:
+        ds_properties = str(Path(root_dir) / "dataset_properties.csv")
+
+    df = load_all_results(root_dir)
+    if Path(ds_properties).exists():
+        df = enrich_with_properties(df, ds_properties)
+
+    for metric in METRIC_COLUMNS:
+        df[metric] = pd.to_numeric(df[metric], errors="coerce")
+
+    return _add_per_dataset_ranks(df)
+
+
 def _add_per_dataset_ranks(df: pd.DataFrame) -> pd.DataFrame:
     df = df.copy()
     group_keys = ["dataset", "term_length", "frequency"]
@@ -155,14 +139,8 @@ def _add_per_dataset_ranks(df: pd.DataFrame) -> pd.DataFrame:
 def get_grouped_dfs(
     root_dir: str = "results",
     ds_properties: str | None = None,
-) -> dict[str, pd.DataFrame]:
-    if ds_properties is None:
-        ds_properties = str(Path(root_dir) / "dataset_properties.csv")
-
-    df = load_all_results(root_dir)
-    df = enrich_with_properties(df, ds_properties)
-    df = standardize_df(df, root_dir)
-    df = _add_per_dataset_ranks(df)
+) -> dict[str, pd.DataFrame | list[str]]:
+    df = prepare_results_df(root_dir, ds_properties)
 
     rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS]
     overall_values = df.groupby(["model"])[DISPLAY_METRICS].agg(stats.gmean)
@@ -173,26 +151,26 @@ def get_grouped_dfs(
         columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS}
     )
 
-    domain_values: dict[str, pd.DataFrame] = {}
-    domain_ranks: dict[str, pd.DataFrame] = {}
-    domains = sorted(df["domain"].dropna().unique()) if "domain" in df.columns else []
-    for domain in domains:
-        subset = df[df["domain"] == domain]
+    datasets = sorted(df["dataset"].dropna().unique())
+    dataset_values: dict[str, pd.DataFrame] = {}
+    dataset_ranks: dict[str, pd.DataFrame] = {}
+    for dataset in datasets:
+        subset = df[df["dataset"] == dataset]
         if subset.empty:
             continue
-        values = subset.groupby(["model"])[DISPLAY_METRICS].agg(stats.gmean)
-        ranks = subset.groupby(["model"])[rank_cols].mean()
-        domain_values[domain] = values.rename(columns=METRIC_LABELS)
-        domain_ranks[domain] = ranks.rename(
+        values = subset.set_index("model")[DISPLAY_METRICS].rename(columns=METRIC_LABELS)
+        ranks = subset.set_index("model")[rank_cols].rename(
             columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS}
         )
+        dataset_values[dataset] = values
+        dataset_ranks[dataset] = ranks
 
     return {
         "overall_values": overall_values,
         "overall_ranks": overall_ranks,
-        "domain_values": domain_values,
-        "domain_ranks": domain_ranks,
-        "domains": domains,
+        "dataset_values": dataset_values,
+        "dataset_ranks": dataset_ranks,
+        "datasets": datasets,
     }
 
 
@@ -210,27 +188,35 @@ def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame:
     return table[cols].sort_values(by=["MASE_Rank"])
 
 
-def prepare_domain_summary(values_by_domain: dict[str, pd.DataFrame]) -> pd.DataFrame:
-    rows = []
-    for domain, values_df in sorted(values_by_domain.items()):
-        for model, metrics in values_df.iterrows():
-            row = {"Domain": domain, "Model": model}
-            row.update(metrics.to_dict())
-            rows.append(row)
-    if not rows:
-        return pd.DataFrame(columns=["Domain", "Model"] + VALUE_COLUMNS)
-    table = pd.DataFrame(rows)
-    return format_df(table.sort_values(by=["Domain", "MASE"]))
-
-
-def prepare_domain_rank_summary(ranks_by_domain: dict[str, pd.DataFrame]) -> pd.DataFrame:
-    rows = []
-    for domain, ranks_df in sorted(ranks_by_domain.items()):
-        for model, metrics in ranks_df.iterrows():
-            row = {"Domain": domain, "Model": model}
-            row.update(metrics.to_dict())
-            rows.append(row)
-    if not rows:
-        return pd.DataFrame(columns=["Domain", "Model"] + RANK_COLUMNS)
-    table = pd.DataFrame(rows)
-    return format_df(table.sort_values(by=["Domain", "MASE_Rank"]))
+def prepare_dataset_pivot(
+    df: pd.DataFrame,
+    metric: str,
+    *,
+    rank: bool = False,
+) -> pd.DataFrame:
+    """Pivot models × datasets (GIFT-Eval style by-domain view)."""
+    if rank:
+        value_col = f"Rank_{metric}"
+        label = f"{METRIC_LABELS[metric]}_Rank"
+    else:
+        value_col = metric
+        label = METRIC_LABELS[metric]
+
+    pivot = df.pivot_table(
+        index="model",
+        columns="dataset",
+        values=value_col,
+        aggfunc="first",
+    )
+    pivot = pivot.reset_index().rename(columns={"model": "Model"})
+    pivot.columns = [
+        "Model" if col == "Model" else str(col)
+        for col in pivot.columns
+    ]
+    pivot.columns.name = None
+    sort_col = "Model" if rank else pivot.columns[1] if len(pivot.columns) > 1 else "Model"
+    if sort_col != "Model" and sort_col in pivot.columns:
+        return format_df(pivot.sort_values(by=sort_col))
+    if rank and "MASE_Rank" in pivot.columns:
+        return format_df(pivot.sort_values(by="MASE_Rank"))
+    return format_df(pivot.sort_values(by="Model"))