from __future__ import annotations import json import os import re from datetime import datetime, timedelta, timezone from pathlib import Path import numpy as np import pandas as pd from scipy import stats from src.display.formatting import format_timestamp_utc8 from src.rank_based_metric import compute_rank_score METRIC_COLUMNS = [ "eval_metrics/MSE[mean]", "eval_metrics/MSE[0.5]", "eval_metrics/MAE[0.5]", "eval_metrics/MASE[0.5]", "eval_metrics/MAPE[0.5]", "eval_metrics/sMAPE[0.5]", "eval_metrics/MSIS", "eval_metrics/RMSE[mean]", "eval_metrics/NRMSE[mean]", "eval_metrics/ND[0.5]", "eval_metrics/mean_weighted_sum_quantile_loss", ] DISPLAY_METRICS = [ "eval_metrics/MSE[mean]", "eval_metrics/mean_weighted_sum_quantile_loss", "eval_metrics/MASE[0.5]", "eval_metrics/MAE[0.5]", "eval_metrics/RMSE[mean]", "eval_metrics/sMAPE[0.5]", "eval_metrics/MSIS", "eval_metrics/ND[0.5]", "eval_metrics/NRMSE[mean]", "eval_metrics/MAPE[0.5]", ] METRIC_LABELS = { "eval_metrics/MSE[mean]": "MSE", "eval_metrics/MASE[0.5]": "MASE", "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", "eval_metrics/MAE[0.5]": "MAE", "eval_metrics/RMSE[mean]": "RMSE", "eval_metrics/sMAPE[0.5]": "sMAPE", "eval_metrics/MSIS": "MSIS", "eval_metrics/ND[0.5]": "ND", "eval_metrics/NRMSE[mean]": "NRMSE", "eval_metrics/MAPE[0.5]": "MAPE", } RANK_SCORE_COLUMN = "RankScore" RANK_SCORE_RANK_COLUMN = "RankScore_Rank" RANK_SCORE_BASE_METRICS = [ "eval_metrics/MSE[mean]", "eval_metrics/mean_weighted_sum_quantile_loss", ] VALUE_COLUMNS = ["MSE", "CRPS", RANK_SCORE_COLUMN] + [ METRIC_LABELS[m] for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"} ] RANK_COLUMNS = ["MSE_Rank", "CRPS_Rank", RANK_SCORE_RANK_COLUMN] + [ f"{METRIC_LABELS[m]}_Rank" for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"} ] LEGACY_MODEL_NAMES = { "TSFM1": "Chronos-Bolt-Tiny", "TSFM2": "Chronos-Bolt-Base", "TSFM3": "Chronos-2", } RESULT_COLUMNS = [ "dataset", "model", *METRIC_COLUMNS, "domain", "num_variates", ] GIFT_AGGREGATE_FILES = { "prediction_length": "results_by_prediction_length.csv", "domain": "results_by_domain.csv", "frequency": "results_by_frequency.csv", } GIFT_AGGREGATE_LABELS = { "prediction_length": "Prediction Length", "domain": "Domain", "frequency": "Frequency", } LIVE_AGGREGATE_FILES = { "overall": "live_overall_up_to_now.csv", "rank": "live_rank_up_to_now.csv", } def format_number(value): """Format a displayed leaderboard metric with a fixed two-decimal precision.""" if pd.isna(value): return np.nan return f"{float(value):.2f}" def aggregate_gmean(series: pd.Series) -> float: values = series.dropna() if values.empty: return np.nan return float(stats.gmean(values)) def aggregate_rank_mean(series: pd.Series) -> float: return float(series.mean(skipna=True)) def format_df(df: pd.DataFrame) -> pd.DataFrame: formatted = df.copy() for col in formatted.columns: if pd.api.types.is_numeric_dtype(formatted[col]): formatted[col] = formatted[col].map(format_number) return formatted def unify_freq(df: pd.DataFrame) -> pd.DataFrame: df = df.copy() df["frequency"] = df["frequency"].str.replace(r"\d+", "", regex=True) df["frequency"] = df["frequency"].str.split("-").str[0] return df def load_all_results(root_dir: str = "results") -> pd.DataFrame: frames = [] for subdir, _, files in os.walk(root_dir): for file in files: if file != "all_results.csv": continue frame = pd.read_csv(os.path.join(subdir, file)) if frame.empty: continue frames.append(frame) if not frames: return pd.DataFrame(columns=RESULT_COLUMNS) df = pd.concat(frames, ignore_index=True) df = df.dropna(subset=["dataset"], how="any") df = df[df["dataset"].astype(str).str.len() > 0] if df.empty: return pd.DataFrame(columns=RESULT_COLUMNS) df = df.sort_values(by=["model", "dataset"]).reset_index(drop=True) df["model"] = df["model"].replace(LEGACY_MODEL_NAMES) parts = df["dataset"].astype(str).str.split("/", expand=True) if parts.shape[1] < 3: for idx in range(parts.shape[1], 3): parts[idx] = "short" if idx == 2 else "" df["dataset"] = parts[0] df["frequency"] = parts[1] df["term_length"] = parts[2] return df def enrich_with_properties(df: pd.DataFrame, ds_properties_path: str) -> pd.DataFrame: props = pd.read_csv(ds_properties_path) props["dataset"] = props["dataset"].str.lower().str.replace(" ", "_") props["dataset"] = props["dataset"].str.replace("-", "_") props["dataset"] = props["dataset"].apply(lambda x: re.sub("_+", "_", x).strip("_")) props_map = props.set_index("dataset").T.to_dict() for dataset_key, values in props_map.items(): for key, value in values.items(): if key == "frequency" and df.loc[df["dataset"] == dataset_key, "frequency"].notna().all(): continue df.loc[df["dataset"] == dataset_key, key] = value return unify_freq(df) def prepare_results_df( root_dir: str = "results", ds_properties: str | None = None, ) -> pd.DataFrame: if ds_properties is None: ds_properties = str(Path(root_dir) / "dataset_properties.csv") df = load_all_results(root_dir) if df.empty: return df if Path(ds_properties).exists(): df = enrich_with_properties(df, ds_properties) for metric in METRIC_COLUMNS: df[metric] = pd.to_numeric(df[metric], errors="coerce") return _add_per_dataset_ranks(df) def _add_per_dataset_ranks(df: pd.DataFrame) -> pd.DataFrame: if df.empty: return df df = df.copy() group_keys = ["dataset", "term_length", "frequency"] for metric in DISPLAY_METRICS: rank_col = f"Rank_{metric}" df[rank_col] = df.groupby(group_keys)[metric].rank(method="first", ascending=True) return df def _empty_grouped_dfs() -> dict[str, pd.DataFrame | list[str]]: rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS] return { "overall_values": pd.DataFrame(columns=DISPLAY_METRICS), "overall_ranks": pd.DataFrame(columns=rank_cols), "dataset_values": {}, "dataset_ranks": {}, "datasets": [], } def get_grouped_dfs( root_dir: str = "results", ds_properties: str | None = None, ) -> dict[str, pd.DataFrame | list[str]]: df = prepare_results_df(root_dir, ds_properties) if df.empty: return _empty_grouped_dfs() rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS] overall_values = df.groupby(["model"])[DISPLAY_METRICS].agg(aggregate_gmean) overall_ranks = df.groupby(["model"])[rank_cols].agg(aggregate_rank_mean) overall_values = overall_values.rename(columns=METRIC_LABELS) overall_ranks = overall_ranks.rename( columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS} ) rank_score = compute_rank_score( df, metric_columns=RANK_SCORE_BASE_METRICS, group_columns=("dataset", "term_length", "frequency"), ) overall_values = overall_values.join(rank_score, how="left") if RANK_SCORE_COLUMN in overall_values.columns: overall_ranks[RANK_SCORE_RANK_COLUMN] = overall_values[RANK_SCORE_COLUMN].rank( method="average", ascending=False, ) datasets = sorted(df["dataset"].dropna().unique()) dataset_values: dict[str, pd.DataFrame] = {} dataset_ranks: dict[str, pd.DataFrame] = {} for dataset in datasets: subset = df[df["dataset"] == dataset] if subset.empty: continue values = subset.set_index("model")[DISPLAY_METRICS].rename(columns=METRIC_LABELS) ranks = subset.set_index("model")[rank_cols].rename( columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS} ) dataset_values[dataset] = values dataset_ranks[dataset] = ranks return { "overall_values": overall_values, "overall_ranks": overall_ranks, "dataset_values": dataset_values, "dataset_ranks": dataset_ranks, "datasets": datasets, } def prepare_values_table(values_df: pd.DataFrame) -> pd.DataFrame: if values_df is None or values_df.empty: return pd.DataFrame(columns=["Model"] + VALUE_COLUMNS) table = values_df.reset_index().rename(columns={"model": "Model"}) cols = ["Model"] + [col for col in VALUE_COLUMNS if col in table.columns] table = table[cols] if "MSE" in table.columns and not table["MSE"].isna().all(): table = table.sort_values(by=["MSE"]) return format_df(table) if "RankScore" in table.columns and not table["RankScore"].isna().all(): table = table.sort_values(by=["RankScore"], ascending=False) return format_df(table) def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame: if ranks_df is None or ranks_df.empty: return pd.DataFrame(columns=["Model"] + RANK_COLUMNS) table = ranks_df.reset_index().rename(columns={"model": "Model"}) cols = ["Model"] + [col for col in RANK_COLUMNS if col in table.columns] table = table[cols] if "MSE_Rank" in table.columns and not table["MSE_Rank"].isna().all(): table = table.sort_values(by=["MSE_Rank"]) return format_df(table) if RANK_SCORE_RANK_COLUMN in table.columns and not table[RANK_SCORE_RANK_COLUMN].isna().all(): table = table.sort_values(by=[RANK_SCORE_RANK_COLUMN]) return format_df(table) def load_gift_aggregate_table( root_dir: str = "results", dimension: str = "prediction_length", ) -> pd.DataFrame: """Load one persisted GIFT-Eval-style grouped result table.""" if dimension not in GIFT_AGGREGATE_FILES: raise ValueError(f"Unknown aggregate dimension: {dimension}") label = GIFT_AGGREGATE_LABELS[dimension] columns = [label, "Model", "MSE", "CRPS", "Rank", "Configs", "Coverage"] path = Path(root_dir) / "aggregates" / GIFT_AGGREGATE_FILES[dimension] if not path.exists(): return pd.DataFrame(columns=columns) try: table = pd.read_csv(path) except (OSError, pd.errors.ParserError, pd.errors.EmptyDataError): return pd.DataFrame(columns=columns) required = { dimension, "model", "MSE", "CRPS", "Rank", "n_configs", "n_group_configs", "coverage", } if table.empty or not required.issubset(table.columns): return pd.DataFrame(columns=columns) table = table.sort_values([dimension, "Rank", "model"]).copy() table["Configs"] = ( table["n_configs"].fillna(0).astype(int).astype(str) + "/" + table["n_group_configs"].fillna(0).astype(int).astype(str) ) table["Coverage"] = table["coverage"].map( lambda value: "n/a" if pd.isna(value) else f"{100.0 * float(value):.2f}%" ) for metric in ("MSE", "CRPS", "Rank"): table[metric] = pd.to_numeric(table[metric], errors="coerce").map(format_number) table = table.rename(columns={dimension: label, "model": "Model"}) return table.loc[:, columns].reset_index(drop=True) def load_gift_aggregate_metadata_md(root_dir: str = "results") -> str: path = Path(root_dir) / "aggregates" / "metadata.json" if not path.exists(): return "Aggregate metadata is not available yet." try: metadata = json.loads(path.read_text()) except (OSError, json.JSONDecodeError): return "Aggregate metadata could not be loaded." generated = format_timestamp_utc8(metadata.get("generated_at", "")) return ( f"**Protocol:** per-configuration normalization against " f"`{metadata.get('baseline_model', 'Seasonal-Naive')}`; geometric mean for " f"MSE/CRPS; mean CRPS rank. **Coverage:** " f"{metadata.get('num_models', 0)} models × {metadata.get('num_configs', 0)} " f"configurations. **Generated:** {generated}." ) def load_live_aggregate_table( root_dir: str = "results", table: str = "overall", ) -> pd.DataFrame: """Load the cumulative future-release metric or rank table.""" if table not in LIVE_AGGREGATE_FILES: raise ValueError(f"Unknown live aggregate table: {table}") if table == "overall": columns = [ "Rank", "Model", "MSE", "RMSE", "MAPE", "CRPS", "RTG", "Stability", "Improvement", "Datasets", "Releases", ] else: columns = [ "Rank", "Model", "Average Rank", "Win Rate", "Elo", "Dataset Pair Matches", "Shared Releases", "Pairwise Matches", ] path = Path(root_dir) / "aggregates" / LIVE_AGGREGATE_FILES[table] if not path.exists(): return pd.DataFrame(columns=columns) try: frame = pd.read_csv(path) except (OSError, pd.errors.ParserError, pd.errors.EmptyDataError): return pd.DataFrame(columns=columns) if frame.empty: return pd.DataFrame(columns=columns) if table == "overall": source_columns = [ "Rank", "model", "MSE", "RMSE", "MAPE", "CRPS", "RTG", "Stability", "Improvement", "n_datasets", "n_releases", ] if not set(source_columns).issubset(frame.columns): return pd.DataFrame(columns=columns) frame = frame[source_columns].rename( columns={"model": "Model", "n_datasets": "Datasets", "n_releases": "Releases"} ) for metric in ("MSE", "RMSE", "MAPE", "CRPS", "RTG", "Stability", "Improvement"): frame[metric] = pd.to_numeric(frame[metric], errors="coerce").map(format_number) else: source_columns = [ "Rank", "model", "AverageRank", "WinRate", "Elo", "dataset_pair_matches", "shared_releases", "pairwise_matches", ] if not set(source_columns).issubset(frame.columns): return pd.DataFrame(columns=columns) frame = frame[source_columns].rename( columns={ "model": "Model", "AverageRank": "Average Rank", "WinRate": "Win Rate", "dataset_pair_matches": "Dataset Pair Matches", "shared_releases": "Shared Releases", "pairwise_matches": "Pairwise Matches", } ) for metric in ("Average Rank", "Win Rate", "Elo"): frame[metric] = pd.to_numeric(frame[metric], errors="coerce").map(format_number) return frame.loc[:, columns].reset_index(drop=True) def load_live_aggregate_metadata_md(root_dir: str = "results") -> str: path = Path(root_dir) / "aggregates" / "live_metadata.json" if not path.exists(): return "Cumulative live metrics are not available yet." try: metadata = json.loads(path.read_text()) except (OSError, json.JSONDecodeError): return "Cumulative live metric metadata could not be loaded." generated = format_timestamp_utc8(metadata.get("generated_at", "")) return ( f"**Up to now:** {metadata.get('num_models', 0)} models, " f"{metadata.get('num_datasets', 0)} datasets, and " f"{metadata.get('num_unique_releases', 0)} distinct future releases. " "MSE/RMSE/CRPS use causal context z-normalization; RTG is relative to " "24-step Seasonal-Naive. **Generated:** " f"{generated}." ) DOMAIN_DISPLAY = { "Weather": "Climate", "Air Quality": "Climate", "Transport": "Transport", "Finance": "Finance", "Ocean": "Ocean", "Hydrology": "Hydrology", "Hazards": "Hazards", "Web": "Web", } DATASET_SHORT_NAMES = { "binance_btcusdt_1h_close": "BTC/USDT", "coingecko_bitcoin_7d_price_usd": "Bitcoin USD", "nasa_power_shanghai_hourly_t2m": "Shanghai temperature (NASA POWER)", "nasa_power_shanghai_hourly_T2M": "Shanghai temperature (NASA POWER)", "open_meteo_shanghai_hourly_temperature_2m": "Shanghai temperature", "open_meteo_shanghai_air_quality_hourly_pm2_5": "Shanghai PM2.5", "noaa_coops_sf_water_level_water_level": "SF Bay water level", "noaa_ndbc_46013_realtime_wave_height": "Ocean wave height", "nws_ksfo_observations_temperature": "KSFO temperature", "usgs_earthquake_all_week_hourly_earthquake_count": "Earthquake count", "usgs_potomac_iv_usgs_00060": "Potomac river flow", "wikimedia_time_series_hourly_pageviews": "Wikipedia pageviews", } def display_domain(raw_domain: str) -> str: return DOMAIN_DISPLAY.get(raw_domain, raw_domain) def dataset_short_name(dataset: str) -> str: base = dataset.split("/")[0].strip() if base in DATASET_SHORT_NAMES: return DATASET_SHORT_NAMES[base] label = base.replace("_", " ") for prefix in ("open meteo ", "noaa coops ", "noaa ndbc ", "nws ", "usgs ", "wikimedia "): if label.lower().startswith(prefix): label = label[len(prefix) :] return label[:40] + ("…" if len(label) > 40 else "") def lookup_dataset_domain(dataset: str, root_dir: str = "results") -> str: # Primary: dataset_properties.csv base = dataset.split("/")[0].strip() props_path = Path(root_dir) / "dataset_properties.csv" if props_path.exists(): props = pd.read_csv(props_path) match = props[props["dataset"].str.lower() == base.lower()] if not match.empty and "domain" in match.columns: return str(match.iloc[0]["domain"]) # Fallback: read domain column from any all_results.csv for subdir in Path(root_dir).iterdir(): csv_path = subdir / "all_results.csv" if not csv_path.exists(): continue try: df = pd.read_csv(csv_path) if "domain" not in df.columns or "dataset" not in df.columns: continue # dataset column in all_results is "dataset_id/freq/term", strip suffix df["_ds_key"] = df["dataset"].astype(str).str.split("/").str[0] match = df[df["_ds_key"].str.lower() == base.lower()] if not match.empty: return str(match.iloc[0]["domain"]) except Exception: continue return "Other" DOMAIN_TAB_ORDER = [ "Finance", "Climate", "Ocean", "Hydrology", "Transport", "Hazards", "Web", "Other", ] def group_datasets_by_domain( datasets: list[str], root_dir: str = "results", ) -> dict[str, list[str]]: by_domain: dict[str, list[str]] = {} for dataset in datasets: domain = display_domain(lookup_dataset_domain(dataset, root_dir)) by_domain.setdefault(domain, []).append(dataset) ordered: dict[str, list[str]] = {} for domain in DOMAIN_TAB_ORDER: if domain in by_domain: ordered[domain] = by_domain.pop(domain) for domain in sorted(by_domain): ordered[domain] = by_domain[domain] return ordered def build_leaderboard_summary_html(root_dir: str = "results") -> str: df = prepare_results_df(root_dir) if df.empty: return """