Fix leaderboard domain and dataset deduplication

src/utils.py

@@ -83,6 +83,35 @@ RESULT_COLUMNS = [
     "num_variates",
 ]
 
+CANONICAL_DOMAINS = [
+    "Finance",
+    "Climate",
+    "Ocean",
+    "Hydrology",
+    "Transportation",
+    "Web",
+    "Hazards",
+]
+
+DOMAIN_DISPLAY = {
+    "Finance": "Finance",
+    "Econ/Fin": "Finance",
+    "Sales": "Finance",
+    "Climate": "Climate",
+    "Weather": "Climate",
+    "Air Quality": "Climate",
+    "Nature": "Climate",
+    "Energy": "Climate",
+    "Ocean": "Ocean",
+    "Hydrology": "Hydrology",
+    "Transport": "Transportation",
+    "Transportation": "Transportation",
+    "Web": "Web",
+    "Web/CloudOps": "Web",
+    "Hazards": "Hazards",
+    "Healthcare": "Hazards",
+}
+
 
 def _read_result_csv(path: str | Path) -> pd.DataFrame:
     df = pd.read_csv(
@@ -204,6 +233,8 @@ def prepare_results_df(
         return df
     if Path(ds_properties).exists():
         df = enrich_with_properties(df, ds_properties)
+    if "domain" in df.columns:
+        df["domain"] = df["domain"].map(display_domain)
 
     for metric in METRIC_COLUMNS:
         df[metric] = pd.to_numeric(df[metric], errors="coerce")
@@ -268,8 +299,10 @@ def get_grouped_dfs(
         subset = df[df["dataset"] == dataset]
         if subset.empty:
             continue
-        values = subset.set_index("model")[DISPLAY_METRICS].rename(columns=METRIC_LABELS)
-        ranks = subset.set_index("model")[rank_cols].rename(
+        values = subset.groupby("model")[DISPLAY_METRICS].agg(aggregate_gmean).rename(
+            columns=METRIC_LABELS
+        )
+        ranks = subset.groupby("model")[rank_cols].agg(aggregate_rank_mean).rename(
             columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS}
         )
         dataset_values[dataset] = values
@@ -488,19 +521,6 @@ def load_live_aggregate_metadata_md(root_dir: str = "results") -> str:
         f"{generated}."
     )
 
-
-DOMAIN_DISPLAY = {
-    "Weather": "Climate",
-    "Air Quality": "Climate",
-    "Transport": "Transport",
-    "Transportation": "Transportation",
-    "Finance": "Finance",
-    "Ocean": "Ocean",
-    "Hydrology": "Hydrology",
-    "Hazards": "Hazards",
-    "Web": "Web",
-}
-
 DATASET_SHORT_NAMES = {
     "binance_btcusdt_1h_close": "BTC/USDT",
     "coingecko_bitcoin_7d_price_usd": "Bitcoin USD",
@@ -521,7 +541,23 @@ DATASET_SHORT_NAMES = {
 
 
 def display_domain(raw_domain: str) -> str:
-    return DOMAIN_DISPLAY.get(raw_domain, raw_domain)
+    domain = str(raw_domain).strip()
+    if domain in DOMAIN_DISPLAY:
+        return DOMAIN_DISPLAY[domain]
+    lowered = domain.lower()
+    if any(token in lowered for token in ("finance", "econ", "sales")):
+        return "Finance"
+    if any(token in lowered for token in ("climate", "weather", "air", "nature", "energy")):
+        return "Climate"
+    if "ocean" in lowered:
+        return "Ocean"
+    if "hydro" in lowered:
+        return "Hydrology"
+    if any(token in lowered for token in ("transport", "taxi")):
+        return "Transportation"
+    if any(token in lowered for token in ("web", "cloud", "wiki")):
+        return "Web"
+    return "Hazards"
 
 
 def dataset_short_name(dataset: str) -> str:
@@ -565,15 +601,7 @@ def lookup_dataset_domain(dataset: str, root_dir: str = "results") -> str:
 
 
 DOMAIN_TAB_ORDER = [
-    "Finance",
-    "Climate",
-    "Ocean",
-    "Hydrology",
-    "Transport",
-    "Transportation",
-    "Hazards",
-    "Web",
-    "Other",
+    *CANONICAL_DOMAINS,
 ]
 
 
@@ -582,7 +610,12 @@ def group_datasets_by_domain(
     root_dir: str = "results",
 ) -> dict[str, list[str]]:
     by_domain: dict[str, list[str]] = {}
+    seen: set[str] = set()
     for dataset in datasets:
+        dataset = dataset.split("/")[0].strip()
+        if dataset in seen:
+            continue
+        seen.add(dataset)
         domain = display_domain(lookup_dataset_domain(dataset, root_dir))
         by_domain.setdefault(domain, []).append(dataset)
 
@@ -889,11 +922,9 @@ DOMAIN_COLORS = {
     "Climate": "#60a5fa",
     "Ocean": "#34d399",
     "Hydrology": "#818cf8",
-    "Transport": "#fbbf24",
     "Transportation": "#14b8a6",
     "Hazards": "#fb7185",
     "Web": "#a78bfa",
-    "Other": "#94a3b8",
 }