LiveHouse-TS / src /utils.py
ziyuzhou02's picture
Deploy Space: update src
0b69772 verified
Raw History Blame
38.2 kB
from __future__ import annotations import json import os import re from datetime import datetime, timedelta, timezone from pathlib import Path import numpy as np import pandas as pd from scipy import stats from src.display.formatting import format_timestamp_utc8 from src.rank_based_metric import compute_rank_score METRIC_COLUMNS = [ "eval_metrics/MSE[mean]", "eval_metrics/MSE[0.5]", "eval_metrics/MAE[0.5]", "eval_metrics/MASE[0.5]", "eval_metrics/MAPE[0.5]", "eval_metrics/sMAPE[0.5]", "eval_metrics/MSIS", "eval_metrics/RMSE[mean]", "eval_metrics/NRMSE[mean]", "eval_metrics/ND[0.5]", "eval_metrics/mean_weighted_sum_quantile_loss", ] DISPLAY_METRICS = [ "eval_metrics/MSE[mean]", "eval_metrics/mean_weighted_sum_quantile_loss", "eval_metrics/MASE[0.5]", "eval_metrics/MAE[0.5]", "eval_metrics/RMSE[mean]", "eval_metrics/sMAPE[0.5]", "eval_metrics/MSIS", "eval_metrics/ND[0.5]", "eval_metrics/NRMSE[mean]", "eval_metrics/MAPE[0.5]", ] METRIC_LABELS = { "eval_metrics/MSE[mean]": "MSE", "eval_metrics/MASE[0.5]": "MASE", "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", "eval_metrics/MAE[0.5]": "MAE", "eval_metrics/RMSE[mean]": "RMSE", "eval_metrics/sMAPE[0.5]": "sMAPE", "eval_metrics/MSIS": "MSIS", "eval_metrics/ND[0.5]": "ND", "eval_metrics/NRMSE[mean]": "NRMSE", "eval_metrics/MAPE[0.5]": "MAPE", } RANK_SCORE_COLUMN = "RankScore" RANK_SCORE_RANK_COLUMN = "RankScore_Rank" RANK_SCORE_BASE_METRICS = [ "eval_metrics/MSE[mean]", "eval_metrics/mean_weighted_sum_quantile_loss", ] VALUE_COLUMNS = ["MSE", "CRPS", RANK_SCORE_COLUMN] + [ METRIC_LABELS[m] for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"} ] RANK_COLUMNS = ["MSE_Rank", "CRPS_Rank", RANK_SCORE_RANK_COLUMN] + [ f"{METRIC_LABELS[m]}_Rank" for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"} ] LEGACY_MODEL_NAMES = { "TSFM1": "Chronos-Bolt-Tiny", "TSFM2": "Chronos-Bolt-Base", "TSFM3": "Chronos-2", } RESULT_COLUMNS = [ "dataset", "model", *METRIC_COLUMNS, "domain", "num_variates", ] GIFT_AGGREGATE_FILES = { "prediction_length": "results_by_prediction_length.csv", "domain": "results_by_domain.csv", "frequency": "results_by_frequency.csv", } GIFT_AGGREGATE_LABELS = { "prediction_length": "Prediction Length", "domain": "Domain", "frequency": "Frequency", } LIVE_AGGREGATE_FILES = { "overall": "live_overall_up_to_now.csv", "rank": "live_rank_up_to_now.csv", } def format_number(value): """Format a displayed leaderboard metric with a fixed two-decimal precision.""" if pd.isna(value): return np.nan return f"{float(value):.2f}" def aggregate_gmean(series: pd.Series) -> float: values = series.dropna() if values.empty: return np.nan return float(stats.gmean(values)) def aggregate_rank_mean(series: pd.Series) -> float: return float(series.mean(skipna=True)) def format_df(df: pd.DataFrame) -> pd.DataFrame: formatted = df.copy() for col in formatted.columns: if pd.api.types.is_numeric_dtype(formatted[col]): formatted[col] = formatted[col].map(format_number) return formatted def unify_freq(df: pd.DataFrame) -> pd.DataFrame: df = df.copy() df["frequency"] = df["frequency"].str.replace(r"\d+", "", regex=True) df["frequency"] = df["frequency"].str.split("-").str[0] return df def load_all_results(root_dir: str = "results") -> pd.DataFrame: frames = [] for subdir, _, files in os.walk(root_dir): for file in files: if file != "all_results.csv": continue frame = pd.read_csv(os.path.join(subdir, file)) if frame.empty: continue frames.append(frame) if not frames: return pd.DataFrame(columns=RESULT_COLUMNS) df = pd.concat(frames, ignore_index=True) df = df.dropna(subset=["dataset"], how="any") df = df[df["dataset"].astype(str).str.len() > 0] if df.empty: return pd.DataFrame(columns=RESULT_COLUMNS) df = df.sort_values(by=["model", "dataset"]).reset_index(drop=True) df["model"] = df["model"].replace(LEGACY_MODEL_NAMES) parts = df["dataset"].astype(str).str.split("/", expand=True) if parts.shape[1] < 3: for idx in range(parts.shape[1], 3): parts[idx] = "short" if idx == 2 else "" df["dataset"] = parts[0] df["frequency"] = parts[1] df["term_length"] = parts[2] return df def enrich_with_properties(df: pd.DataFrame, ds_properties_path: str) -> pd.DataFrame: props = pd.read_csv(ds_properties_path) props["dataset"] = props["dataset"].str.lower().str.replace(" ", "_") props["dataset"] = props["dataset"].str.replace("-", "_") props["dataset"] = props["dataset"].apply(lambda x: re.sub("_+", "_", x).strip("_")) props_map = props.set_index("dataset").T.to_dict() for dataset_key, values in props_map.items(): for key, value in values.items(): if key == "frequency" and df.loc[df["dataset"] == dataset_key, "frequency"].notna().all(): continue df.loc[df["dataset"] == dataset_key, key] = value return unify_freq(df) def prepare_results_df( root_dir: str = "results", ds_properties: str | None = None, ) -> pd.DataFrame: if ds_properties is None: ds_properties = str(Path(root_dir) / "dataset_properties.csv") df = load_all_results(root_dir) if df.empty: return df if Path(ds_properties).exists(): df = enrich_with_properties(df, ds_properties) for metric in METRIC_COLUMNS: df[metric] = pd.to_numeric(df[metric], errors="coerce") return _add_per_dataset_ranks(df) def _add_per_dataset_ranks(df: pd.DataFrame) -> pd.DataFrame: if df.empty: return df df = df.copy() group_keys = ["dataset", "term_length", "frequency"] for metric in DISPLAY_METRICS: rank_col = f"Rank_{metric}" df[rank_col] = df.groupby(group_keys)[metric].rank(method="first", ascending=True) return df def _empty_grouped_dfs() -> dict[str, pd.DataFrame | list[str]]: rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS] return { "overall_values": pd.DataFrame(columns=DISPLAY_METRICS), "overall_ranks": pd.DataFrame(columns=rank_cols), "dataset_values": {}, "dataset_ranks": {}, "datasets": [], } def get_grouped_dfs( root_dir: str = "results", ds_properties: str | None = None, ) -> dict[str, pd.DataFrame | list[str]]: df = prepare_results_df(root_dir, ds_properties) if df.empty: return _empty_grouped_dfs() rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS] overall_values = df.groupby(["model"])[DISPLAY_METRICS].agg(aggregate_gmean) overall_ranks = df.groupby(["model"])[rank_cols].agg(aggregate_rank_mean) overall_values = overall_values.rename(columns=METRIC_LABELS) overall_ranks = overall_ranks.rename( columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS} ) rank_score = compute_rank_score( df, metric_columns=RANK_SCORE_BASE_METRICS, group_columns=("dataset", "term_length", "frequency"), ) overall_values = overall_values.join(rank_score, how="left") if RANK_SCORE_COLUMN in overall_values.columns: overall_ranks[RANK_SCORE_RANK_COLUMN] = overall_values[RANK_SCORE_COLUMN].rank( method="average", ascending=False, ) datasets = sorted(df["dataset"].dropna().unique()) dataset_values: dict[str, pd.DataFrame] = {} dataset_ranks: dict[str, pd.DataFrame] = {} for dataset in datasets: subset = df[df["dataset"] == dataset] if subset.empty: continue values = subset.set_index("model")[DISPLAY_METRICS].rename(columns=METRIC_LABELS) ranks = subset.set_index("model")[rank_cols].rename( columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS} ) dataset_values[dataset] = values dataset_ranks[dataset] = ranks return { "overall_values": overall_values, "overall_ranks": overall_ranks, "dataset_values": dataset_values, "dataset_ranks": dataset_ranks, "datasets": datasets, } def prepare_values_table(values_df: pd.DataFrame) -> pd.DataFrame: if values_df is None or values_df.empty: return pd.DataFrame(columns=["Model"] + VALUE_COLUMNS) table = values_df.reset_index().rename(columns={"model": "Model"}) cols = ["Model"] + [col for col in VALUE_COLUMNS if col in table.columns] table = table[cols] if "MSE" in table.columns and not table["MSE"].isna().all(): table = table.sort_values(by=["MSE"]) return format_df(table) if "RankScore" in table.columns and not table["RankScore"].isna().all(): table = table.sort_values(by=["RankScore"], ascending=False) return format_df(table) def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame: if ranks_df is None or ranks_df.empty: return pd.DataFrame(columns=["Model"] + RANK_COLUMNS) table = ranks_df.reset_index().rename(columns={"model": "Model"}) cols = ["Model"] + [col for col in RANK_COLUMNS if col in table.columns] table = table[cols] if "MSE_Rank" in table.columns and not table["MSE_Rank"].isna().all(): table = table.sort_values(by=["MSE_Rank"]) return format_df(table) if RANK_SCORE_RANK_COLUMN in table.columns and not table[RANK_SCORE_RANK_COLUMN].isna().all(): table = table.sort_values(by=[RANK_SCORE_RANK_COLUMN]) return format_df(table) def load_gift_aggregate_table( root_dir: str = "results", dimension: str = "prediction_length", ) -> pd.DataFrame: """Load one persisted GIFT-Eval-style grouped result table.""" if dimension not in GIFT_AGGREGATE_FILES: raise ValueError(f"Unknown aggregate dimension: {dimension}") label = GIFT_AGGREGATE_LABELS[dimension] columns = [label, "Model", "MSE", "CRPS", "Rank", "Configs", "Coverage"] path = Path(root_dir) / "aggregates" / GIFT_AGGREGATE_FILES[dimension] if not path.exists(): return pd.DataFrame(columns=columns) try: table = pd.read_csv(path) except (OSError, pd.errors.ParserError, pd.errors.EmptyDataError): return pd.DataFrame(columns=columns) required = { dimension, "model", "MSE", "CRPS", "Rank", "n_configs", "n_group_configs", "coverage", } if table.empty or not required.issubset(table.columns): return pd.DataFrame(columns=columns) table = table.sort_values([dimension, "Rank", "model"]).copy() table["Configs"] = ( table["n_configs"].fillna(0).astype(int).astype(str) + "/" + table["n_group_configs"].fillna(0).astype(int).astype(str) ) table["Coverage"] = table["coverage"].map( lambda value: "n/a" if pd.isna(value) else f"{100.0 * float(value):.2f}%" ) for metric in ("MSE", "CRPS", "Rank"): table[metric] = pd.to_numeric(table[metric], errors="coerce").map(format_number) table = table.rename(columns={dimension: label, "model": "Model"}) return table.loc[:, columns].reset_index(drop=True) def load_gift_aggregate_metadata_md(root_dir: str = "results") -> str: path = Path(root_dir) / "aggregates" / "metadata.json" if not path.exists(): return "Aggregate metadata is not available yet." try: metadata = json.loads(path.read_text()) except (OSError, json.JSONDecodeError): return "Aggregate metadata could not be loaded." generated = format_timestamp_utc8(metadata.get("generated_at", "")) return ( f"**Protocol:** per-configuration normalization against " f"`{metadata.get('baseline_model', 'Seasonal-Naive')}`; geometric mean for " f"MSE/CRPS; mean CRPS rank. **Coverage:** " f"{metadata.get('num_models', 0)} models × {metadata.get('num_configs', 0)} " f"configurations. **Generated:** {generated}." ) def load_live_aggregate_table( root_dir: str = "results", table: str = "overall", ) -> pd.DataFrame: """Load the cumulative future-release metric or rank table.""" if table not in LIVE_AGGREGATE_FILES: raise ValueError(f"Unknown live aggregate table: {table}") if table == "overall": columns = [ "Rank", "Model", "MSE", "RMSE", "MAPE", "CRPS", "RTG", "Stability", "Improvement", "Datasets", "Releases", ] else: columns = [ "Rank", "Model", "Average Rank", "Win Rate", "Elo", "Dataset Pair Matches", "Shared Releases", "Pairwise Matches", ] path = Path(root_dir) / "aggregates" / LIVE_AGGREGATE_FILES[table] if not path.exists(): return pd.DataFrame(columns=columns) try: frame = pd.read_csv(path) except (OSError, pd.errors.ParserError, pd.errors.EmptyDataError): return pd.DataFrame(columns=columns) if frame.empty: return pd.DataFrame(columns=columns) if table == "overall": source_columns = [ "Rank", "model", "MSE", "RMSE", "MAPE", "CRPS", "RTG", "Stability", "Improvement", "n_datasets", "n_releases", ] if not set(source_columns).issubset(frame.columns): return pd.DataFrame(columns=columns) frame = frame[source_columns].rename( columns={"model": "Model", "n_datasets": "Datasets", "n_releases": "Releases"} ) for metric in ("MSE", "RMSE", "MAPE", "CRPS", "RTG", "Stability", "Improvement"): frame[metric] = pd.to_numeric(frame[metric], errors="coerce").map(format_number) else: source_columns = [ "Rank", "model", "AverageRank", "WinRate", "Elo", "dataset_pair_matches", "shared_releases", "pairwise_matches", ] if not set(source_columns).issubset(frame.columns): return pd.DataFrame(columns=columns) frame = frame[source_columns].rename( columns={ "model": "Model", "AverageRank": "Average Rank", "WinRate": "Win Rate", "dataset_pair_matches": "Dataset Pair Matches", "shared_releases": "Shared Releases", "pairwise_matches": "Pairwise Matches", } ) for metric in ("Average Rank", "Win Rate", "Elo"): frame[metric] = pd.to_numeric(frame[metric], errors="coerce").map(format_number) return frame.loc[:, columns].reset_index(drop=True) def load_live_aggregate_metadata_md(root_dir: str = "results") -> str: path = Path(root_dir) / "aggregates" / "live_metadata.json" if not path.exists(): return "Cumulative live metrics are not available yet." try: metadata = json.loads(path.read_text()) except (OSError, json.JSONDecodeError): return "Cumulative live metric metadata could not be loaded." generated = format_timestamp_utc8(metadata.get("generated_at", "")) return ( f"**Up to now:** {metadata.get('num_models', 0)} models, " f"{metadata.get('num_datasets', 0)} datasets, and " f"{metadata.get('num_unique_releases', 0)} distinct future releases. " "MSE/RMSE/CRPS use causal context z-normalization; RTG is relative to " "24-step Seasonal-Naive. **Generated:** " f"{generated}." ) DOMAIN_DISPLAY = { "Weather": "Climate", "Air Quality": "Climate", "Transport": "Transport", "Finance": "Finance", "Ocean": "Ocean", "Hydrology": "Hydrology", "Hazards": "Hazards", "Web": "Web", } DATASET_SHORT_NAMES = { "binance_btcusdt_1h_close": "BTC/USDT", "coingecko_bitcoin_7d_price_usd": "Bitcoin USD", "nasa_power_shanghai_hourly_t2m": "Shanghai temperature (NASA POWER)", "nasa_power_shanghai_hourly_T2M": "Shanghai temperature (NASA POWER)", "open_meteo_shanghai_hourly_temperature_2m": "Shanghai temperature", "open_meteo_shanghai_air_quality_hourly_pm2_5": "Shanghai PM2.5", "noaa_coops_sf_water_level_water_level": "SF Bay water level", "noaa_ndbc_46013_realtime_wave_height": "Ocean wave height", "nws_ksfo_observations_temperature": "KSFO temperature", "usgs_earthquake_all_week_hourly_earthquake_count": "Earthquake count", "usgs_potomac_iv_usgs_00060": "Potomac river flow", "wikimedia_time_series_hourly_pageviews": "Wikipedia pageviews", } def display_domain(raw_domain: str) -> str: return DOMAIN_DISPLAY.get(raw_domain, raw_domain) def dataset_short_name(dataset: str) -> str: base = dataset.split("/")[0].strip() if base in DATASET_SHORT_NAMES: return DATASET_SHORT_NAMES[base] label = base.replace("_", " ") for prefix in ("open meteo ", "noaa coops ", "noaa ndbc ", "nws ", "usgs ", "wikimedia "): if label.lower().startswith(prefix): label = label[len(prefix) :] return label[:40] + ("…" if len(label) > 40 else "") def lookup_dataset_domain(dataset: str, root_dir: str = "results") -> str: # Primary: dataset_properties.csv base = dataset.split("/")[0].strip() props_path = Path(root_dir) / "dataset_properties.csv" if props_path.exists(): props = pd.read_csv(props_path) match = props[props["dataset"].str.lower() == base.lower()] if not match.empty and "domain" in match.columns: return str(match.iloc[0]["domain"]) # Fallback: read domain column from any all_results.csv for subdir in Path(root_dir).iterdir(): csv_path = subdir / "all_results.csv" if not csv_path.exists(): continue try: df = pd.read_csv(csv_path) if "domain" not in df.columns or "dataset" not in df.columns: continue # dataset column in all_results is "dataset_id/freq/term", strip suffix df["_ds_key"] = df["dataset"].astype(str).str.split("/").str[0] match = df[df["_ds_key"].str.lower() == base.lower()] if not match.empty: return str(match.iloc[0]["domain"]) except Exception: continue return "Other" DOMAIN_TAB_ORDER = [ "Finance", "Climate", "Ocean", "Hydrology", "Transport", "Hazards", "Web", "Other", ] def group_datasets_by_domain( datasets: list[str], root_dir: str = "results", ) -> dict[str, list[str]]: by_domain: dict[str, list[str]] = {} for dataset in datasets: domain = display_domain(lookup_dataset_domain(dataset, root_dir)) by_domain.setdefault(domain, []).append(dataset) ordered: dict[str, list[str]] = {} for domain in DOMAIN_TAB_ORDER: if domain in by_domain: ordered[domain] = by_domain.pop(domain) for domain in sorted(by_domain): ordered[domain] = by_domain[domain] return ordered def build_leaderboard_summary_html(root_dir: str = "results") -> str: df = prepare_results_df(root_dir) if df.empty: return """ <div class="summary-grid"> <div class="summary-card"><div class="summary-value">0</div><div class="summary-label">Models</div></div> <div class="summary-card"><div class="summary-value">0</div><div class="summary-label">Datasets</div></div> <div class="summary-card"><div class="summary-value">0</div><div class="summary-label">Domains</div></div> <div class="summary-card"><div class="summary-value">n/a</div><div class="summary-label">Last Refresh</div></div> </div> """ models = df["model"].nunique() datasets = df["dataset"].nunique() domains = df["domain"].nunique() if "domain" in df.columns else 0 last_refresh = "n/a" status_path = Path(root_dir) / "online_status.json" if status_path.exists(): try: status = json.loads(status_path.read_text()) last_refresh = format_timestamp_utc8(status.get("finished_at", "")) except Exception: pass return f""" <div class="summary-grid"> <div class="summary-card"><div class="summary-value">{models}</div><div class="summary-label">Models</div></div> <div class="summary-card"><div class="summary-value">{datasets}</div><div class="summary-label">Datasets</div></div> <div class="summary-card"><div class="summary-value">{domains}</div><div class="summary-label">Domains</div></div> <div class="summary-card wide"><div class="summary-value small">{last_refresh}</div><div class="summary-label">Last Refresh</div></div> </div> """ def dataset_tab_label(dataset: str, root_dir: str = "results") -> str: raw_domain = lookup_dataset_domain(dataset, root_dir) domain = display_domain(raw_domain) short = dataset_short_name(dataset) return f"{domain} · {short}" def dataset_section_title(dataset: str, root_dir: str, *, ranks: bool = False) -> str: label = dataset_tab_label(dataset, root_dir) if ranks: return f"**{label} · ranks (lower is better)**" return f"**{label} · metric values (absolute, lower is better)**" BASELINE_RANK_HISTORY_COLUMNS = [ "date", "model", "MSE_Rank", "CRPS_Rank", "MASE_Rank", "MAE_Rank", "RMSE_Rank", "sMAPE_Rank", "MSIS_Rank", "ND_Rank", "NRMSE_Rank", "MAPE_Rank", ] _RANK_COLS = [c for c in BASELINE_RANK_HISTORY_COLUMNS if c.endswith("_Rank")] _DAILY_DISPLAY_COLS = ["date"] + _RANK_COLS _WEEKLY_DISPLAY_COLS = ["week"] + _RANK_COLS def _read_rank_history(path: Path, date_col: str) -> pd.DataFrame: """Read a rank history CSV, coerce rank columns to float, sort by date_col.""" if not path.exists(): return pd.DataFrame(columns=[date_col] + _RANK_COLS) try: df = pd.read_csv(path) except Exception: return pd.DataFrame(columns=[date_col] + _RANK_COLS) for col in _RANK_COLS: if col in df.columns: df[col] = pd.to_numeric(df[col], errors="coerce").map(format_number) if date_col in df.columns: df = df.sort_values(date_col) present = [c for c in [date_col] + _RANK_COLS if c in df.columns] return df[present].reset_index(drop=True) def load_baseline_rank_history(root_dir: str = "results") -> pd.DataFrame: """Load daily baseline rank history (baseline_rank_history_daily.csv).""" # Also try the legacy single-file name for backward-compat for name in ("baseline_rank_history_daily.csv", "baseline_rank_history.csv"): path = Path(root_dir) / name if path.exists(): return _read_rank_history(path, "date") return pd.DataFrame(columns=_DAILY_DISPLAY_COLS) def load_baseline_rank_history_weekly(root_dir: str = "results") -> pd.DataFrame: """Load weekly aggregated baseline rank history (baseline_rank_history_weekly.csv).""" return _read_rank_history(Path(root_dir) / "baseline_rank_history_weekly.csv", "week") # --------------------------------------------------------------------------- # Forecast snapshot utilities # --------------------------------------------------------------------------- def load_forecast_snapshots(root_dir: str, model_slug: str) -> dict[str, dict]: """Load all forecast JSON files for a given model slug. Returns a dict keyed by the dataset display name (from snapshot["dataset"]). """ forecasts_dir = Path(root_dir) / model_slug / "forecasts" snapshots: dict[str, dict] = {} if not forecasts_dir.exists(): return snapshots for fp in sorted(forecasts_dir.glob("*.json")): try: snap = json.loads(fp.read_text()) snapshots[snap.get("dataset", fp.stem)] = snap except Exception: pass return snapshots def _get_plotly(): """Lazy import plotly to avoid import errors in non-chart contexts.""" try: import plotly.graph_objects as go return go except ImportError: return None def make_forecast_plot(snapshot: dict | None) -> object | None: """Build a plotly Figure showing context + actuals vs p50 forecast with p10/p90 band. Returns None if snapshot is None or plotly is unavailable. """ go = _get_plotly() if go is None or snapshot is None: return None ctx = snapshot.get("context", []) actuals = snapshot.get("actuals", []) p50 = snapshot.get("p50", []) p10 = snapshot.get("p10", []) p90 = snapshot.get("p90", []) n_ctx = len(ctx) n_fut = len(actuals) timestamps = snapshot.get("timestamps", []) if timestamps and len(timestamps) == n_ctx + n_fut: ctx_x = timestamps[:n_ctx] fut_x = timestamps[n_ctx:] x_title = "Time" vline_x = ctx_x[-1] if n_ctx > 0 else fut_x[0] else: ctx_x = list(range(n_ctx)) fut_x = list(range(n_ctx, n_ctx + n_fut)) x_title = "Time step" vline_x = n_ctx - 1.0 if n_ctx > 0 else 0 if n_ctx > 0: fut_x = [ctx_x[-1]] + list(fut_x) actuals = [ctx[-1]] + list(actuals) p50 = [ctx[-1]] + list(p50) if p50 else [] p10 = [ctx[-1]] + list(p10) if p10 else [] p90 = [ctx[-1]] + list(p90) if p90 else [] n_fut = len(actuals) fig = go.Figure() # Context (historical) fig.add_trace(go.Scatter( x=ctx_x, y=ctx, mode="lines", name="Context (history)", line=dict(color="#94a3b8", width=1.5), )) # Actuals (ground truth for forecast horizon) fig.add_trace(go.Scatter( x=fut_x, y=actuals, mode="lines", name="Actual", line=dict(color="#3b82f6", width=2), )) # p10-p90 confidence band if p10 and p90 and len(p10) == n_fut and len(p90) == n_fut: fig.add_trace(go.Scatter( x=fut_x + fut_x[::-1], y=p90 + p10[::-1], fill="toself", fillcolor="rgba(239,68,68,0.15)", line=dict(color="rgba(255,255,255,0)"), name="p10-p90 band", showlegend=True, )) # p50 forecast if p50 and len(p50) == n_fut: fig.add_trace(go.Scatter( x=fut_x, y=p50, mode="lines", name="Forecast (p50)", line=dict(color="#ef4444", width=2, dash="dash"), )) dataset_label = snapshot.get("dataset", "").split("/")[0] model_label = snapshot.get("model", "") evaluated_at = snapshot.get("evaluated_at", "")[:10] fig.update_layout( title=dict( text=f"{dataset_label} — {model_label} ({evaluated_at})", font=dict(size=14), ), xaxis_title=x_title, yaxis_title="Value", legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), margin=dict(l=40, r=20, t=60, b=40), height=350, paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", font=dict(color="black"), xaxis=dict(gridcolor="rgba(0,0,0,0.08)"), yaxis=dict(gridcolor="rgba(0,0,0,0.08)"), ) # Vertical line separating context from forecast fig.add_vline( x=vline_x, line_dash="dot", line_color="rgba(0,0,0,0.3)", annotation_text="forecast start", annotation_font_color="black", ) return fig def make_rank_trend_plot(history_df: pd.DataFrame, metrics: list[str] | None = None) -> object | None: """Build a plotly Figure showing selected rank metrics over time (date on X axis). Args: history_df: DataFrame from load_baseline_rank_history() with a 'date' column. metrics: List of rank column names to plot. Defaults to MSE_Rank and CRPS_Rank. Returns None if fewer than 1 data point or plotly unavailable. """ go = _get_plotly() if go is None or history_df is None or history_df.empty: return None if metrics is None: metrics = ["MSE_Rank", "CRPS_Rank", "MASE_Rank", "MAE_Rank"] date_col = "date" if "date" in history_df.columns else "week" present_metrics = [m for m in metrics if m in history_df.columns] if not present_metrics: return None COLORS = ["#f97316", "#a78bfa", "#34d399", "#60a5fa", "#fb7185", "#fbbf24"] fig = go.Figure() for i, col in enumerate(present_metrics): fig.add_trace(go.Scatter( x=history_df[date_col], y=history_df[col], mode="lines+markers", name=col.replace("_Rank", ""), line=dict(color=COLORS[i % len(COLORS)], width=2), marker=dict(size=6), )) fig.update_layout( title=dict(text="Baseline rank trend over time (lower is better)", font=dict(size=14)), xaxis_title=date_col.capitalize(), yaxis_title="Average rank", legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), margin=dict(l=40, r=20, t=60, b=40), height=320, paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", font=dict(color="black"), xaxis=dict(gridcolor="rgba(0,0,0,0.08)"), yaxis=dict(gridcolor="rgba(0,0,0,0.08)"), ) return fig DOMAIN_COLORS = { "Finance": "#f97316", "Climate": "#60a5fa", "Ocean": "#34d399", "Hydrology": "#818cf8", "Transport": "#fbbf24", "Hazards": "#fb7185", "Web": "#a78bfa", "Other": "#94a3b8", } def _today_start_utc8() -> datetime: """Return midnight today in UTC+8 as an aware datetime.""" utc8 = timezone(timedelta(hours=8)) return datetime.now(utc8).replace(hour=0, minute=0, second=0, microsecond=0) def get_dataset_last_eval_time(ds_base: str, root_dir: str = "results") -> datetime | None: latest_dt: datetime | None = None # Scan model directories for model_dir in Path(root_dir).iterdir(): if not model_dir.is_dir(): continue forecasts_dir = model_dir / "forecasts" if not forecasts_dir.is_dir(): continue # Find files matching the base name for fp in forecasts_dir.glob(f"{ds_base}*.json"): try: # To be super fast, regex search for evaluated_at content = fp.read_text() match = re.search(r'"evaluated_at"\s*:\s*"([^"]+)"', content) if match: eval_at = match.group(1) dt = datetime.fromisoformat(eval_at.replace("Z", "+00:00")) if latest_dt is None or dt > latest_dt: latest_dt = dt except Exception: continue return latest_dt def make_domain_pie_chart(root_dir: str = "results") -> object | None: """Build a plotly Pie chart showing evaluated timestamp count per domain today. Reads eval_history.jsonl to count total evaluated timestamps since UTC+8 00:00 today, grouped by domain. """ go = _get_plotly() if go is None: return None # Domain mapping from dataset_properties.csv props_path = Path(root_dir) / "dataset_properties.csv" if not props_path.exists(): return None try: props = pd.read_csv(props_path) except Exception: return None domain_map: dict[str, str] = {} for _, row in props.iterrows(): domain_map[str(row["dataset"]).lower()] = display_domain(str(row["domain"])) def get_domain(ds_name: str) -> str: ds_lower = ds_name.lower() if ds_lower not in domain_map: domain_map[ds_lower] = display_domain(lookup_dataset_domain(ds_name, root_dir)) return domain_map[ds_lower] today_start = _today_start_utc8() utc8 = today_start.tzinfo # Read eval_history.jsonl (Cumulative) history_path = Path(root_dir) / "eval_history.jsonl" timestamp_counts: dict[str, int] = {} # domain → total timestamps domain_datasets: dict[str, set[str]] = {} # domain → set of dataset short names if history_path.exists(): try: with open(history_path) as f: for line in f: if not line.strip(): continue entry = json.loads(line) ds_base = entry.get("dataset", "") pred_len = entry.get("prediction_length", 0) domain = get_domain(ds_base) timestamp_counts[domain] = timestamp_counts.get(domain, 0) + pred_len domain_datasets.setdefault(domain, set()).add(dataset_short_name(ds_base)) except Exception: pass # Fallback to cumulative snapshots count (no today filter) if empty if not timestamp_counts: seen_datasets: set[str] = set() for model_dir in sorted(Path(root_dir).iterdir()): if not model_dir.is_dir(): continue forecasts_dir = model_dir / "forecasts" if not forecasts_dir.is_dir(): continue for fp in forecasts_dir.glob("*.json"): try: snap = json.loads(fp.read_text()) except Exception: continue ds_full = snap.get("dataset", "") ds_base = ds_full.split("/")[0] if ds_base in seen_datasets: continue seen_datasets.add(ds_base) domain = get_domain(ds_base) n_ts = len(snap.get("context", [])) + len(snap.get("actuals", [])) timestamp_counts[domain] = timestamp_counts.get(domain, 0) + n_ts domain_datasets.setdefault(domain, set()).add(dataset_short_name(ds_base)) if not timestamp_counts: return None labels = list(timestamp_counts.keys()) values = list(timestamp_counts.values()) hover = [", ".join(sorted(domain_datasets.get(d, []))) for d in labels] colors = [DOMAIN_COLORS.get(label, "#94a3b8") for label in labels] fig = go.Figure(data=[go.Pie( labels=labels, values=values, hovertext=hover, hoverinfo="label+percent+text", textinfo="label+value+percent", textposition="inside", textfont=dict(size=14, color="white"), insidetextorientation="radial", marker=dict(colors=colors), hole=0.25, )]) fig.update_layout( title=dict( text="Evaluated timestamps by domain", font=dict(size=16, weight="bold"), ), margin=dict(l=20, r=20, t=60, b=20), height=500, paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", font=dict(color="black"), showlegend=True, legend=dict( orientation="h", yanchor="bottom", y=-0.1, xanchor="center", x=0.5 ) ) return fig # --------------------------------------------------------------------------- # Per-domain status helpers # --------------------------------------------------------------------------- def load_eval_intervals(root_dir: str = "results") -> dict[str, int]: """Load per-dataset eval intervals from dataset_eval_intervals.json.""" path = Path(root_dir) / "dataset_eval_intervals.json" if not path.exists(): return {} try: return {str(k): int(v) for k, v in json.loads(path.read_text()).items()} except Exception: return {} def _format_interval(seconds: int) -> str: if seconds >= 86400: d = seconds // 86400 return f"{d}d" if seconds >= 3600: h = seconds // 3600 return f"{h}h" if seconds >= 60: m = seconds // 60 return f"{m}min" return f"{seconds}s" def build_domain_status_md( domain: str, datasets: list[str], root_dir: str = "results", ) -> str: """Build a compact markdown status line for a domain tab. Shows data source, last eval time, last evaluated time for this domain, and per-dataset refresh intervals. """ utc8 = timezone(timedelta(hours=8)) # 1. Load data source name from online_status.json status_path = Path(root_dir) / "online_status.json" data_source = "TS-Bench" if status_path.exists(): try: status = json.loads(status_path.read_text()) data_source = status.get("data_source", "TS-Bench") except Exception: pass # 2. Last evaluated time for this domain (max of all datasets in the domain) domain_latest_dt: datetime | None = None for ds in datasets: ds_base = ds.split("/")[0].strip() eval_dt = get_dataset_last_eval_time(ds_base, root_dir) if eval_dt: if domain_latest_dt is None or eval_dt > domain_latest_dt: domain_latest_dt = eval_dt last_evaluated = "n/a" if domain_latest_dt: last_evaluated = domain_latest_dt.astimezone(utc8).strftime("%Y-%m-%d %H:%M:%S") + " (UTC+8)" # 3. Auto-refresh per-dataset intervals = load_eval_intervals(root_dir) ds_parts: list[str] = [] for ds in datasets: short = dataset_short_name(ds) ds_base = ds.split("/")[0].strip() interval = intervals.get(ds_base) if interval: ds_parts.append(f"{short} every {_format_interval(interval)}") else: ds_parts.append(short) parts = [ f"**Data**: {data_source}", f"**Last eval**: {last_evaluated}", ] if ds_parts: parts.append("**Refresh**: " + " · ".join(ds_parts)) return " | ".join(parts)
from __future__ import annotations
import json
import os
import re
from datetime import datetime, timedelta, timezone
from pathlib import Path
import numpy as np
import pandas as pd
from scipy import stats
from src.display.formatting import format_timestamp_utc8
from src.rank_based_metric import compute_rank_score
METRIC_COLUMNS = [
"eval_metrics/MSE[mean]",
"eval_metrics/MSE[0.5]",
"eval_metrics/MAE[0.5]",
"eval_metrics/MASE[0.5]",
"eval_metrics/MAPE[0.5]",
"eval_metrics/sMAPE[0.5]",
"eval_metrics/MSIS",
"eval_metrics/RMSE[mean]",
"eval_metrics/NRMSE[mean]",
"eval_metrics/ND[0.5]",
"eval_metrics/mean_weighted_sum_quantile_loss",
]
DISPLAY_METRICS = [
"eval_metrics/MSE[mean]",
"eval_metrics/mean_weighted_sum_quantile_loss",
"eval_metrics/MASE[0.5]",
"eval_metrics/MAE[0.5]",
"eval_metrics/RMSE[mean]",
"eval_metrics/sMAPE[0.5]",
"eval_metrics/MSIS",
"eval_metrics/ND[0.5]",
"eval_metrics/NRMSE[mean]",
"eval_metrics/MAPE[0.5]",
]
METRIC_LABELS = {
"eval_metrics/MSE[mean]": "MSE",
"eval_metrics/MASE[0.5]": "MASE",
"eval_metrics/mean_weighted_sum_quantile_loss": "CRPS",
"eval_metrics/MAE[0.5]": "MAE",
"eval_metrics/RMSE[mean]": "RMSE",
"eval_metrics/sMAPE[0.5]": "sMAPE",
"eval_metrics/MSIS": "MSIS",
"eval_metrics/ND[0.5]": "ND",
"eval_metrics/NRMSE[mean]": "NRMSE",
"eval_metrics/MAPE[0.5]": "MAPE",
}
RANK_SCORE_COLUMN = "RankScore"
RANK_SCORE_RANK_COLUMN = "RankScore_Rank"
RANK_SCORE_BASE_METRICS = [
"eval_metrics/MSE[mean]",
"eval_metrics/mean_weighted_sum_quantile_loss",
]
VALUE_COLUMNS = ["MSE", "CRPS", RANK_SCORE_COLUMN] + [
METRIC_LABELS[m] for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"}
]
RANK_COLUMNS = ["MSE_Rank", "CRPS_Rank", RANK_SCORE_RANK_COLUMN] + [
f"{METRIC_LABELS[m]}_Rank"
for m in DISPLAY_METRICS
if METRIC_LABELS[m] not in {"MSE", "CRPS"}
]
LEGACY_MODEL_NAMES = {
"TSFM1": "Chronos-Bolt-Tiny",
"TSFM2": "Chronos-Bolt-Base",
"TSFM3": "Chronos-2",
}
RESULT_COLUMNS = [
"dataset",
"model",
*METRIC_COLUMNS,
"domain",
"num_variates",
]
GIFT_AGGREGATE_FILES = {
"prediction_length": "results_by_prediction_length.csv",
"domain": "results_by_domain.csv",
"frequency": "results_by_frequency.csv",
}
GIFT_AGGREGATE_LABELS = {
"prediction_length": "Prediction Length",
"domain": "Domain",
"frequency": "Frequency",
}
LIVE_AGGREGATE_FILES = {
"overall": "live_overall_up_to_now.csv",
"rank": "live_rank_up_to_now.csv",