Spaces:
Runtime error
Runtime error
Download src/utils.py from ThinkcatLab/LiveHouse-TS: direct link, hf CLI and curl.
- Browser
- Download file 31.8 kB
-
https://huggingface.co/spaces/ThinkcatLab/LiveHouse-TS/resolve/64c0a2c3daccd2de4ff1bc26df8dfd6fa5982968/src/utils.py
- Command line
-
hf download hf://spaces/ThinkcatLab/LiveHouse-TS@64c0a2c3daccd2de4ff1bc26df8dfd6fa5982968/src/utils.py
-
curl -L -o utils.py https://huggingface.co/spaces/ThinkcatLab/LiveHouse-TS/resolve/64c0a2c3daccd2de4ff1bc26df8dfd6fa5982968/src/utils.py
31.8 kB
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| from datetime import datetime, timedelta, timezone | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| from scipy import stats | |
| from src.display.formatting import format_timestamp_utc8 | |
| from src.rank_based_metric import compute_rank_score | |
| METRIC_COLUMNS = [ | |
| "eval_metrics/MSE[mean]", | |
| "eval_metrics/MSE[0.5]", | |
| "eval_metrics/MAE[0.5]", | |
| "eval_metrics/MASE[0.5]", | |
| "eval_metrics/MAPE[0.5]", | |
| "eval_metrics/sMAPE[0.5]", | |
| "eval_metrics/MSIS", | |
| "eval_metrics/RMSE[mean]", | |
| "eval_metrics/NRMSE[mean]", | |
| "eval_metrics/ND[0.5]", | |
| "eval_metrics/mean_weighted_sum_quantile_loss", | |
| ] | |
| DISPLAY_METRICS = [ | |
| "eval_metrics/MSE[mean]", | |
| "eval_metrics/mean_weighted_sum_quantile_loss", | |
| "eval_metrics/MASE[0.5]", | |
| "eval_metrics/MAE[0.5]", | |
| "eval_metrics/RMSE[mean]", | |
| "eval_metrics/sMAPE[0.5]", | |
| "eval_metrics/MSIS", | |
| "eval_metrics/ND[0.5]", | |
| "eval_metrics/NRMSE[mean]", | |
| "eval_metrics/MAPE[0.5]", | |
| ] | |
| METRIC_LABELS = { | |
| "eval_metrics/MSE[mean]": "MSE", | |
| "eval_metrics/MASE[0.5]": "MASE", | |
| "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", | |
| "eval_metrics/MAE[0.5]": "MAE", | |
| "eval_metrics/RMSE[mean]": "RMSE", | |
| "eval_metrics/sMAPE[0.5]": "sMAPE", | |
| "eval_metrics/MSIS": "MSIS", | |
| "eval_metrics/ND[0.5]": "ND", | |
| "eval_metrics/NRMSE[mean]": "NRMSE", | |
| "eval_metrics/MAPE[0.5]": "MAPE", | |
| } | |
| RANK_SCORE_COLUMN = "RankScore" | |
| RANK_SCORE_RANK_COLUMN = "RankScore_Rank" | |
| RANK_SCORE_BASE_METRICS = [ | |
| "eval_metrics/MSE[mean]", | |
| "eval_metrics/mean_weighted_sum_quantile_loss", | |
| ] | |
| VALUE_COLUMNS = ["MSE", "CRPS", RANK_SCORE_COLUMN] + [ | |
| METRIC_LABELS[m] for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"} | |
| ] | |
| RANK_COLUMNS = ["MSE_Rank", "CRPS_Rank", RANK_SCORE_RANK_COLUMN] + [ | |
| f"{METRIC_LABELS[m]}_Rank" | |
| for m in DISPLAY_METRICS | |
| if METRIC_LABELS[m] not in {"MSE", "CRPS"} | |
| ] | |
| LEGACY_MODEL_NAMES = { | |
| "TSFM1": "Chronos-Bolt-Tiny", | |
| "TSFM2": "Chronos-Bolt-Base", | |
| "TSFM3": "Chronos-2", | |
| } | |
| RESULT_COLUMNS = [ | |
| "dataset", | |
| "model", | |
| *METRIC_COLUMNS, | |
| "domain", | |
| "num_variates", | |
| ] | |
| def format_number(value): | |
| if isinstance(value, (int, float)): | |
| if pd.isna(value): | |
| return np.nan | |
| if abs(value) >= 10**2: | |
| return f"{value:.1e}" | |
| return f"{value:.3f}" | |
| return value | |
| def aggregate_gmean(series: pd.Series) -> float: | |
| values = series.dropna() | |
| if values.empty: | |
| return np.nan | |
| return float(stats.gmean(values)) | |
| def aggregate_rank_mean(series: pd.Series) -> float: | |
| return float(series.mean(skipna=True)) | |
| def format_df(df: pd.DataFrame) -> pd.DataFrame: | |
| formatted = df.copy() | |
| for col in formatted.columns: | |
| if pd.api.types.is_numeric_dtype(formatted[col]): | |
| if col == RANK_SCORE_COLUMN: | |
| formatted[col] = formatted[col].map(lambda x: np.nan if pd.isna(x) else f"{x:.1f}") | |
| else: | |
| formatted[col] = formatted[col].map(format_number) | |
| formatted[col] = formatted[col].astype(float) | |
| return formatted | |
| def unify_freq(df: pd.DataFrame) -> pd.DataFrame: | |
| df = df.copy() | |
| df["frequency"] = df["frequency"].str.replace(r"\d+", "", regex=True) | |
| df["frequency"] = df["frequency"].str.split("-").str[0] | |
| return df | |
| def load_all_results(root_dir: str = "results") -> pd.DataFrame: | |
| frames = [] | |
| for subdir, _, files in os.walk(root_dir): | |
| for file in files: | |
| if file != "all_results.csv": | |
| continue | |
| frame = pd.read_csv(os.path.join(subdir, file)) | |
| if frame.empty: | |
| continue | |
| frames.append(frame) | |
| if not frames: | |
| return pd.DataFrame(columns=RESULT_COLUMNS) | |
| df = pd.concat(frames, ignore_index=True) | |
| df = df.dropna(subset=["dataset"], how="any") | |
| df = df[df["dataset"].astype(str).str.len() > 0] | |
| if df.empty: | |
| return pd.DataFrame(columns=RESULT_COLUMNS) | |
| df = df.sort_values(by=["model", "dataset"]).reset_index(drop=True) | |
| df["model"] = df["model"].replace(LEGACY_MODEL_NAMES) | |
| parts = df["dataset"].astype(str).str.split("/", expand=True) | |
| if parts.shape[1] < 3: | |
| for idx in range(parts.shape[1], 3): | |
| parts[idx] = "short" if idx == 2 else "" | |
| df["dataset"] = parts[0] | |
| df["frequency"] = parts[1] | |
| df["term_length"] = parts[2] | |
| return df | |
| def enrich_with_properties(df: pd.DataFrame, ds_properties_path: str) -> pd.DataFrame: | |
| props = pd.read_csv(ds_properties_path) | |
| props["dataset"] = props["dataset"].str.lower().str.replace(" ", "_") | |
| props["dataset"] = props["dataset"].str.replace("-", "_") | |
| props["dataset"] = props["dataset"].apply(lambda x: re.sub("_+", "_", x).strip("_")) | |
| props_map = props.set_index("dataset").T.to_dict() | |
| for dataset_key, values in props_map.items(): | |
| for key, value in values.items(): | |
| if key == "frequency" and df.loc[df["dataset"] == dataset_key, "frequency"].notna().all(): | |
| continue | |
| df.loc[df["dataset"] == dataset_key, key] = value | |
| return unify_freq(df) | |
| def prepare_results_df( | |
| root_dir: str = "results", | |
| ds_properties: str | None = None, | |
| ) -> pd.DataFrame: | |
| if ds_properties is None: | |
| ds_properties = str(Path(root_dir) / "dataset_properties.csv") | |
| df = load_all_results(root_dir) | |
| if df.empty: | |
| return df | |
| if Path(ds_properties).exists(): | |
| df = enrich_with_properties(df, ds_properties) | |
| for metric in METRIC_COLUMNS: | |
| df[metric] = pd.to_numeric(df[metric], errors="coerce") | |
| return _add_per_dataset_ranks(df) | |
| def _add_per_dataset_ranks(df: pd.DataFrame) -> pd.DataFrame: | |
| if df.empty: | |
| return df | |
| df = df.copy() | |
| group_keys = ["dataset", "term_length", "frequency"] | |
| for metric in DISPLAY_METRICS: | |
| rank_col = f"Rank_{metric}" | |
| df[rank_col] = df.groupby(group_keys)[metric].rank(method="first", ascending=True) | |
| return df | |
| def _empty_grouped_dfs() -> dict[str, pd.DataFrame | list[str]]: | |
| rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS] | |
| return { | |
| "overall_values": pd.DataFrame(columns=DISPLAY_METRICS), | |
| "overall_ranks": pd.DataFrame(columns=rank_cols), | |
| "dataset_values": {}, | |
| "dataset_ranks": {}, | |
| "datasets": [], | |
| } | |
| def get_grouped_dfs( | |
| root_dir: str = "results", | |
| ds_properties: str | None = None, | |
| ) -> dict[str, pd.DataFrame | list[str]]: | |
| df = prepare_results_df(root_dir, ds_properties) | |
| if df.empty: | |
| return _empty_grouped_dfs() | |
| rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS] | |
| overall_values = df.groupby(["model"])[DISPLAY_METRICS].agg(aggregate_gmean) | |
| overall_ranks = df.groupby(["model"])[rank_cols].agg(aggregate_rank_mean) | |
| overall_values = overall_values.rename(columns=METRIC_LABELS) | |
| overall_ranks = overall_ranks.rename( | |
| columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS} | |
| ) | |
| rank_score = compute_rank_score( | |
| df, | |
| metric_columns=RANK_SCORE_BASE_METRICS, | |
| group_columns=("dataset", "term_length", "frequency"), | |
| ) | |
| overall_values = overall_values.join(rank_score, how="left") | |
| if RANK_SCORE_COLUMN in overall_values.columns: | |
| overall_ranks[RANK_SCORE_RANK_COLUMN] = overall_values[RANK_SCORE_COLUMN].rank( | |
| method="average", | |
| ascending=False, | |
| ) | |
| datasets = sorted(df["dataset"].dropna().unique()) | |
| dataset_values: dict[str, pd.DataFrame] = {} | |
| dataset_ranks: dict[str, pd.DataFrame] = {} | |
| for dataset in datasets: | |
| subset = df[df["dataset"] == dataset] | |
| if subset.empty: | |
| continue | |
| values = subset.set_index("model")[DISPLAY_METRICS].rename(columns=METRIC_LABELS) | |
| ranks = subset.set_index("model")[rank_cols].rename( | |
| columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS} | |
| ) | |
| dataset_values[dataset] = values | |
| dataset_ranks[dataset] = ranks | |
| return { | |
| "overall_values": overall_values, | |
| "overall_ranks": overall_ranks, | |
| "dataset_values": dataset_values, | |
| "dataset_ranks": dataset_ranks, | |
| "datasets": datasets, | |
| } | |
| def prepare_values_table(values_df: pd.DataFrame) -> pd.DataFrame: | |
| if values_df is None or values_df.empty: | |
| return pd.DataFrame(columns=["Model"] + VALUE_COLUMNS) | |
| table = values_df.reset_index().rename(columns={"model": "Model"}) | |
| cols = ["Model"] + [col for col in VALUE_COLUMNS if col in table.columns] | |
| table = table[cols] | |
| if "MSE" in table.columns and not table["MSE"].isna().all(): | |
| table = table.sort_values(by=["MSE"]) | |
| return format_df(table) | |
| if "RankScore" in table.columns and not table["RankScore"].isna().all(): | |
| table = table.sort_values(by=["RankScore"], ascending=False) | |
| return format_df(table) | |
| def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame: | |
| if ranks_df is None or ranks_df.empty: | |
| return pd.DataFrame(columns=["Model"] + RANK_COLUMNS) | |
| table = ranks_df.reset_index().rename(columns={"model": "Model"}) | |
| cols = ["Model"] + [col for col in RANK_COLUMNS if col in table.columns] | |
| table = table[cols] | |
| if "MSE_Rank" in table.columns and not table["MSE_Rank"].isna().all(): | |
| table = table.sort_values(by=["MSE_Rank"]) | |
| return format_df(table) | |
| if RANK_SCORE_RANK_COLUMN in table.columns and not table[RANK_SCORE_RANK_COLUMN].isna().all(): | |
| table = table.sort_values(by=[RANK_SCORE_RANK_COLUMN]) | |
| return format_df(table) | |
| DOMAIN_DISPLAY = { | |
| "Weather": "Climate", | |
| "Air Quality": "Climate", | |
| "Transport": "Transport", | |
| "Finance": "Finance", | |
| "Ocean": "Ocean", | |
| "Hydrology": "Hydrology", | |
| "Hazards": "Hazards", | |
| "Web": "Web", | |
| } | |
| DATASET_SHORT_NAMES = { | |
| "binance_btcusdt_1h_close": "BTC/USDT", | |
| "coingecko_bitcoin_7d_price_usd": "Bitcoin USD", | |
| "nasa_power_shanghai_hourly_t2m": "Shanghai temperature (NASA POWER)", | |
| "nasa_power_shanghai_hourly_T2M": "Shanghai temperature (NASA POWER)", | |
| "open_meteo_shanghai_hourly_temperature_2m": "Shanghai temperature", | |
| "open_meteo_shanghai_air_quality_hourly_pm2_5": "Shanghai PM2.5", | |
| "noaa_coops_sf_water_level_water_level": "SF Bay water level", | |
| "noaa_ndbc_46013_realtime_wave_height": "Ocean wave height", | |
| "nws_ksfo_observations_temperature": "KSFO temperature", | |
| "usgs_earthquake_all_week_hourly_earthquake_count": "Earthquake count", | |
| "usgs_potomac_iv_usgs_00060": "Potomac river flow", | |
| "wikimedia_time_series_hourly_pageviews": "Wikipedia pageviews", | |
| } | |
| def display_domain(raw_domain: str) -> str: | |
| return DOMAIN_DISPLAY.get(raw_domain, raw_domain) | |
| def dataset_short_name(dataset: str) -> str: | |
| base = dataset.split("/")[0].strip() | |
| if base in DATASET_SHORT_NAMES: | |
| return DATASET_SHORT_NAMES[base] | |
| label = base.replace("_", " ") | |
| for prefix in ("open meteo ", "noaa coops ", "noaa ndbc ", "nws ", "usgs ", "wikimedia "): | |
| if label.lower().startswith(prefix): | |
| label = label[len(prefix) :] | |
| return label[:40] + ("…" if len(label) > 40 else "") | |
| def lookup_dataset_domain(dataset: str, root_dir: str = "results") -> str: | |
| # Primary: dataset_properties.csv | |
| base = dataset.split("/")[0].strip() | |
| props_path = Path(root_dir) / "dataset_properties.csv" | |
| if props_path.exists(): | |
| props = pd.read_csv(props_path) | |
| match = props[props["dataset"].str.lower() == base.lower()] | |
| if not match.empty and "domain" in match.columns: | |
| return str(match.iloc[0]["domain"]) | |
| # Fallback: read domain column from any all_results.csv | |
| for subdir in Path(root_dir).iterdir(): | |
| csv_path = subdir / "all_results.csv" | |
| if not csv_path.exists(): | |
| continue | |
| try: | |
| df = pd.read_csv(csv_path) | |
| if "domain" not in df.columns or "dataset" not in df.columns: | |
| continue | |
| # dataset column in all_results is "dataset_id/freq/term", strip suffix | |
| df["_ds_key"] = df["dataset"].astype(str).str.split("/").str[0] | |
| match = df[df["_ds_key"].str.lower() == base.lower()] | |
| if not match.empty: | |
| return str(match.iloc[0]["domain"]) | |
| except Exception: | |
| continue | |
| return "Other" | |
| DOMAIN_TAB_ORDER = [ | |
| "Finance", | |
| "Climate", | |
| "Ocean", | |
| "Hydrology", | |
| "Transport", | |
| "Hazards", | |
| "Web", | |
| "Other", | |
| ] | |
| def group_datasets_by_domain( | |
| datasets: list[str], | |
| root_dir: str = "results", | |
| ) -> dict[str, list[str]]: | |
| by_domain: dict[str, list[str]] = {} | |
| for dataset in datasets: | |
| domain = display_domain(lookup_dataset_domain(dataset, root_dir)) | |
| by_domain.setdefault(domain, []).append(dataset) | |
| ordered: dict[str, list[str]] = {} | |
| for domain in DOMAIN_TAB_ORDER: | |
| if domain in by_domain: | |
| ordered[domain] = by_domain.pop(domain) | |
| for domain in sorted(by_domain): | |
| ordered[domain] = by_domain[domain] | |
| return ordered | |
| def build_leaderboard_summary_html(root_dir: str = "results") -> str: | |
| df = prepare_results_df(root_dir) | |
| if df.empty: | |
| return """ | |
| <div class="summary-grid"> | |
| <div class="summary-card"><div class="summary-value">0</div><div class="summary-label">Models</div></div> | |
| <div class="summary-card"><div class="summary-value">0</div><div class="summary-label">Datasets</div></div> | |
| <div class="summary-card"><div class="summary-value">0</div><div class="summary-label">Domains</div></div> | |
| <div class="summary-card"><div class="summary-value">n/a</div><div class="summary-label">Last Refresh</div></div> | |
| </div> | |
| """ | |
| models = df["model"].nunique() | |
| datasets = df["dataset"].nunique() | |
| domains = df["domain"].nunique() if "domain" in df.columns else 0 | |
| last_refresh = "n/a" | |
| status_path = Path(root_dir) / "online_status.json" | |
| if status_path.exists(): | |
| try: | |
| status = json.loads(status_path.read_text()) | |
| last_refresh = format_timestamp_utc8(status.get("finished_at", "")) | |
| except Exception: | |
| pass | |
| return f""" | |
| <div class="summary-grid"> | |
| <div class="summary-card"><div class="summary-value">{models}</div><div class="summary-label">Models</div></div> | |
| <div class="summary-card"><div class="summary-value">{datasets}</div><div class="summary-label">Datasets</div></div> | |
| <div class="summary-card"><div class="summary-value">{domains}</div><div class="summary-label">Domains</div></div> | |
| <div class="summary-card wide"><div class="summary-value small">{last_refresh}</div><div class="summary-label">Last Refresh</div></div> | |
| </div> | |
| """ | |
| def dataset_tab_label(dataset: str, root_dir: str = "results") -> str: | |
| raw_domain = lookup_dataset_domain(dataset, root_dir) | |
| domain = display_domain(raw_domain) | |
| short = dataset_short_name(dataset) | |
| return f"{domain} · {short}" | |
| def dataset_section_title(dataset: str, root_dir: str, *, ranks: bool = False) -> str: | |
| label = dataset_tab_label(dataset, root_dir) | |
| if ranks: | |
| return f"**{label} · ranks (lower is better)**" | |
| return f"**{label} · metric values (absolute, lower is better)**" | |
| BASELINE_RANK_HISTORY_COLUMNS = [ | |
| "date", | |
| "model", | |
| "MSE_Rank", | |
| "CRPS_Rank", | |
| "MASE_Rank", | |
| "MAE_Rank", | |
| "RMSE_Rank", | |
| "sMAPE_Rank", | |
| "MSIS_Rank", | |
| "ND_Rank", | |
| "NRMSE_Rank", | |
| "MAPE_Rank", | |
| ] | |
| _RANK_COLS = [c for c in BASELINE_RANK_HISTORY_COLUMNS if c.endswith("_Rank")] | |
| _DAILY_DISPLAY_COLS = ["date"] + _RANK_COLS | |
| _WEEKLY_DISPLAY_COLS = ["week"] + _RANK_COLS | |
| def _read_rank_history(path: Path, date_col: str) -> pd.DataFrame: | |
| """Read a rank history CSV, coerce rank columns to float, sort by date_col.""" | |
| if not path.exists(): | |
| return pd.DataFrame(columns=[date_col] + _RANK_COLS) | |
| try: | |
| df = pd.read_csv(path) | |
| except Exception: | |
| return pd.DataFrame(columns=[date_col] + _RANK_COLS) | |
| for col in _RANK_COLS: | |
| if col in df.columns: | |
| df[col] = pd.to_numeric(df[col], errors="coerce").round(2) | |
| if date_col in df.columns: | |
| df = df.sort_values(date_col) | |
| present = [c for c in [date_col] + _RANK_COLS if c in df.columns] | |
| return df[present].reset_index(drop=True) | |
| def load_baseline_rank_history(root_dir: str = "results") -> pd.DataFrame: | |
| """Load daily baseline rank history (baseline_rank_history_daily.csv).""" | |
| # Also try the legacy single-file name for backward-compat | |
| for name in ("baseline_rank_history_daily.csv", "baseline_rank_history.csv"): | |
| path = Path(root_dir) / name | |
| if path.exists(): | |
| return _read_rank_history(path, "date") | |
| return pd.DataFrame(columns=_DAILY_DISPLAY_COLS) | |
| def load_baseline_rank_history_weekly(root_dir: str = "results") -> pd.DataFrame: | |
| """Load weekly aggregated baseline rank history (baseline_rank_history_weekly.csv).""" | |
| return _read_rank_history(Path(root_dir) / "baseline_rank_history_weekly.csv", "week") | |
| # --------------------------------------------------------------------------- | |
| # Forecast snapshot utilities | |
| # --------------------------------------------------------------------------- | |
| def load_forecast_snapshots(root_dir: str, model_slug: str) -> dict[str, dict]: | |
| """Load all forecast JSON files for a given model slug. | |
| Returns a dict keyed by the dataset display name (from snapshot["dataset"]). | |
| """ | |
| forecasts_dir = Path(root_dir) / model_slug / "forecasts" | |
| snapshots: dict[str, dict] = {} | |
| if not forecasts_dir.exists(): | |
| return snapshots | |
| for fp in sorted(forecasts_dir.glob("*.json")): | |
| try: | |
| snap = json.loads(fp.read_text()) | |
| snapshots[snap.get("dataset", fp.stem)] = snap | |
| except Exception: | |
| pass | |
| return snapshots | |
| def _get_plotly(): | |
| """Lazy import plotly to avoid import errors in non-chart contexts.""" | |
| try: | |
| import plotly.graph_objects as go | |
| return go | |
| except ImportError: | |
| return None | |
| def make_forecast_plot(snapshot: dict | None) -> object | None: | |
| """Build a plotly Figure showing context + actuals vs p50 forecast with p10/p90 band. | |
| Returns None if snapshot is None or plotly is unavailable. | |
| """ | |
| go = _get_plotly() | |
| if go is None or snapshot is None: | |
| return None | |
| ctx = snapshot.get("context", []) | |
| actuals = snapshot.get("actuals", []) | |
| p50 = snapshot.get("p50", []) | |
| p10 = snapshot.get("p10", []) | |
| p90 = snapshot.get("p90", []) | |
| n_ctx = len(ctx) | |
| n_fut = len(actuals) | |
| timestamps = snapshot.get("timestamps", []) | |
| if timestamps and len(timestamps) == n_ctx + n_fut: | |
| ctx_x = timestamps[:n_ctx] | |
| fut_x = timestamps[n_ctx:] | |
| x_title = "Time" | |
| vline_x = ctx_x[-1] if n_ctx > 0 else fut_x[0] | |
| else: | |
| ctx_x = list(range(n_ctx)) | |
| fut_x = list(range(n_ctx, n_ctx + n_fut)) | |
| x_title = "Time step" | |
| vline_x = n_ctx - 1.0 if n_ctx > 0 else 0 | |
| if n_ctx > 0: | |
| fut_x = [ctx_x[-1]] + list(fut_x) | |
| actuals = [ctx[-1]] + list(actuals) | |
| p50 = [ctx[-1]] + list(p50) if p50 else [] | |
| p10 = [ctx[-1]] + list(p10) if p10 else [] | |
| p90 = [ctx[-1]] + list(p90) if p90 else [] | |
| n_fut = len(actuals) | |
| fig = go.Figure() | |
| # Context (historical) | |
| fig.add_trace(go.Scatter( | |
| x=ctx_x, y=ctx, | |
| mode="lines", | |
| name="Context (history)", | |
| line=dict(color="#94a3b8", width=1.5), | |
| )) | |
| # Actuals (ground truth for forecast horizon) | |
| fig.add_trace(go.Scatter( | |
| x=fut_x, y=actuals, | |
| mode="lines", | |
| name="Actual", | |
| line=dict(color="#3b82f6", width=2), | |
| )) | |
| # p10-p90 confidence band | |
| if p10 and p90 and len(p10) == n_fut and len(p90) == n_fut: | |
| fig.add_trace(go.Scatter( | |
| x=fut_x + fut_x[::-1], | |
| y=p90 + p10[::-1], | |
| fill="toself", | |
| fillcolor="rgba(239,68,68,0.15)", | |
| line=dict(color="rgba(255,255,255,0)"), | |
| name="p10-p90 band", | |
| showlegend=True, | |
| )) | |
| # p50 forecast | |
| if p50 and len(p50) == n_fut: | |
| fig.add_trace(go.Scatter( | |
| x=fut_x, y=p50, | |
| mode="lines", | |
| name="Forecast (p50)", | |
| line=dict(color="#ef4444", width=2, dash="dash"), | |
| )) | |
| dataset_label = snapshot.get("dataset", "").split("/")[0] | |
| model_label = snapshot.get("model", "") | |
| evaluated_at = snapshot.get("evaluated_at", "")[:10] | |
| fig.update_layout( | |
| title=dict( | |
| text=f"{dataset_label} — {model_label} ({evaluated_at})", | |
| font=dict(size=14), | |
| ), | |
| xaxis_title=x_title, | |
| yaxis_title="Value", | |
| legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), | |
| margin=dict(l=40, r=20, t=60, b=40), | |
| height=350, | |
| paper_bgcolor="rgba(0,0,0,0)", | |
| plot_bgcolor="rgba(0,0,0,0)", | |
| font=dict(color="black"), | |
| xaxis=dict(gridcolor="rgba(0,0,0,0.08)"), | |
| yaxis=dict(gridcolor="rgba(0,0,0,0.08)"), | |
| ) | |
| # Vertical line separating context from forecast | |
| fig.add_vline( | |
| x=vline_x, | |
| line_dash="dot", | |
| line_color="rgba(0,0,0,0.3)", | |
| annotation_text="forecast start", | |
| annotation_font_color="black", | |
| ) | |
| return fig | |
| def make_rank_trend_plot(history_df: pd.DataFrame, metrics: list[str] | None = None) -> object | None: | |
| """Build a plotly Figure showing selected rank metrics over time (date on X axis). | |
| Args: | |
| history_df: DataFrame from load_baseline_rank_history() with a 'date' column. | |
| metrics: List of rank column names to plot. Defaults to MSE_Rank and CRPS_Rank. | |
| Returns None if fewer than 1 data point or plotly unavailable. | |
| """ | |
| go = _get_plotly() | |
| if go is None or history_df is None or history_df.empty: | |
| return None | |
| if metrics is None: | |
| metrics = ["MSE_Rank", "CRPS_Rank", "MASE_Rank", "MAE_Rank"] | |
| date_col = "date" if "date" in history_df.columns else "week" | |
| present_metrics = [m for m in metrics if m in history_df.columns] | |
| if not present_metrics: | |
| return None | |
| COLORS = ["#f97316", "#a78bfa", "#34d399", "#60a5fa", "#fb7185", "#fbbf24"] | |
| fig = go.Figure() | |
| for i, col in enumerate(present_metrics): | |
| fig.add_trace(go.Scatter( | |
| x=history_df[date_col], | |
| y=history_df[col], | |
| mode="lines+markers", | |
| name=col.replace("_Rank", ""), | |
| line=dict(color=COLORS[i % len(COLORS)], width=2), | |
| marker=dict(size=6), | |
| )) | |
| fig.update_layout( | |
| title=dict(text="Baseline rank trend over time (lower is better)", font=dict(size=14)), | |
| xaxis_title=date_col.capitalize(), | |
| yaxis_title="Average rank", | |
| legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), | |
| margin=dict(l=40, r=20, t=60, b=40), | |
| height=320, | |
| paper_bgcolor="rgba(0,0,0,0)", | |
| plot_bgcolor="rgba(0,0,0,0)", | |
| font=dict(color="black"), | |
| xaxis=dict(gridcolor="rgba(0,0,0,0.08)"), | |
| yaxis=dict(gridcolor="rgba(0,0,0,0.08)"), | |
| ) | |
| return fig | |
| DOMAIN_COLORS = { | |
| "Finance": "#f97316", | |
| "Climate": "#60a5fa", | |
| "Ocean": "#34d399", | |
| "Hydrology": "#818cf8", | |
| "Transport": "#fbbf24", | |
| "Hazards": "#fb7185", | |
| "Web": "#a78bfa", | |
| "Other": "#94a3b8", | |
| } | |
| def _today_start_utc8() -> datetime: | |
| """Return midnight today in UTC+8 as an aware datetime.""" | |
| utc8 = timezone(timedelta(hours=8)) | |
| return datetime.now(utc8).replace(hour=0, minute=0, second=0, microsecond=0) | |
| def get_dataset_last_eval_time(ds_base: str, root_dir: str = "results") -> datetime | None: | |
| latest_dt: datetime | None = None | |
| # Scan model directories | |
| for model_dir in Path(root_dir).iterdir(): | |
| if not model_dir.is_dir(): | |
| continue | |
| forecasts_dir = model_dir / "forecasts" | |
| if not forecasts_dir.is_dir(): | |
| continue | |
| # Find files matching the base name | |
| for fp in forecasts_dir.glob(f"{ds_base}*.json"): | |
| try: | |
| # To be super fast, regex search for evaluated_at | |
| content = fp.read_text() | |
| match = re.search(r'"evaluated_at"\s*:\s*"([^"]+)"', content) | |
| if match: | |
| eval_at = match.group(1) | |
| dt = datetime.fromisoformat(eval_at.replace("Z", "+00:00")) | |
| if latest_dt is None or dt > latest_dt: | |
| latest_dt = dt | |
| except Exception: | |
| continue | |
| return latest_dt | |
| def make_domain_pie_chart(root_dir: str = "results") -> object | None: | |
| """Build a plotly Pie chart showing evaluated timestamp count per domain today. | |
| Reads eval_history.jsonl to count total evaluated timestamps since UTC+8 00:00 today, | |
| grouped by domain. | |
| """ | |
| go = _get_plotly() | |
| if go is None: | |
| return None | |
| # Domain mapping from dataset_properties.csv | |
| props_path = Path(root_dir) / "dataset_properties.csv" | |
| if not props_path.exists(): | |
| return None | |
| try: | |
| props = pd.read_csv(props_path) | |
| except Exception: | |
| return None | |
| domain_map: dict[str, str] = {} | |
| for _, row in props.iterrows(): | |
| domain_map[str(row["dataset"]).lower()] = display_domain(str(row["domain"])) | |
| def get_domain(ds_name: str) -> str: | |
| ds_lower = ds_name.lower() | |
| if ds_lower not in domain_map: | |
| domain_map[ds_lower] = display_domain(lookup_dataset_domain(ds_name, root_dir)) | |
| return domain_map[ds_lower] | |
| today_start = _today_start_utc8() | |
| utc8 = today_start.tzinfo | |
| # Read eval_history.jsonl (Cumulative) | |
| history_path = Path(root_dir) / "eval_history.jsonl" | |
| timestamp_counts: dict[str, int] = {} # domain → total timestamps | |
| domain_datasets: dict[str, set[str]] = {} # domain → set of dataset short names | |
| if history_path.exists(): | |
| try: | |
| with open(history_path) as f: | |
| for line in f: | |
| if not line.strip(): | |
| continue | |
| entry = json.loads(line) | |
| ds_base = entry.get("dataset", "") | |
| pred_len = entry.get("prediction_length", 0) | |
| domain = get_domain(ds_base) | |
| timestamp_counts[domain] = timestamp_counts.get(domain, 0) + pred_len | |
| domain_datasets.setdefault(domain, set()).add(dataset_short_name(ds_base)) | |
| except Exception: | |
| pass | |
| # Fallback to cumulative snapshots count (no today filter) if empty | |
| if not timestamp_counts: | |
| seen_datasets: set[str] = set() | |
| for model_dir in sorted(Path(root_dir).iterdir()): | |
| if not model_dir.is_dir(): | |
| continue | |
| forecasts_dir = model_dir / "forecasts" | |
| if not forecasts_dir.is_dir(): | |
| continue | |
| for fp in forecasts_dir.glob("*.json"): | |
| try: | |
| snap = json.loads(fp.read_text()) | |
| except Exception: | |
| continue | |
| ds_full = snap.get("dataset", "") | |
| ds_base = ds_full.split("/")[0] | |
| if ds_base in seen_datasets: | |
| continue | |
| seen_datasets.add(ds_base) | |
| domain = get_domain(ds_base) | |
| n_ts = len(snap.get("context", [])) + len(snap.get("actuals", [])) | |
| timestamp_counts[domain] = timestamp_counts.get(domain, 0) + n_ts | |
| domain_datasets.setdefault(domain, set()).add(dataset_short_name(ds_base)) | |
| if not timestamp_counts: | |
| return None | |
| labels = list(timestamp_counts.keys()) | |
| values = list(timestamp_counts.values()) | |
| hover = [", ".join(sorted(domain_datasets.get(d, []))) for d in labels] | |
| colors = [DOMAIN_COLORS.get(label, "#94a3b8") for label in labels] | |
| fig = go.Figure(data=[go.Pie( | |
| labels=labels, | |
| values=values, | |
| hovertext=hover, | |
| hoverinfo="label+percent+text", | |
| textinfo="label+value+percent", | |
| textposition="inside", | |
| textfont=dict(size=14, color="white"), | |
| insidetextorientation="radial", | |
| marker=dict(colors=colors), | |
| hole=0.25, | |
| )]) | |
| fig.update_layout( | |
| title=dict( | |
| text="Evaluated timestamps by domain", | |
| font=dict(size=16, weight="bold"), | |
| ), | |
| margin=dict(l=20, r=20, t=60, b=20), | |
| height=500, | |
| paper_bgcolor="rgba(0,0,0,0)", | |
| plot_bgcolor="rgba(0,0,0,0)", | |
| font=dict(color="black"), | |
| showlegend=True, | |
| legend=dict( | |
| orientation="h", | |
| yanchor="bottom", | |
| y=-0.1, | |
| xanchor="center", | |
| x=0.5 | |
| ) | |
| ) | |
| return fig | |
| # --------------------------------------------------------------------------- | |
| # Per-domain status helpers | |
| # --------------------------------------------------------------------------- | |
| def load_eval_intervals(root_dir: str = "results") -> dict[str, int]: | |
| """Load per-dataset eval intervals from dataset_eval_intervals.json.""" | |
| path = Path(root_dir) / "dataset_eval_intervals.json" | |
| if not path.exists(): | |
| return {} | |
| try: | |
| return {str(k): int(v) for k, v in json.loads(path.read_text()).items()} | |
| except Exception: | |
| return {} | |
| def _format_interval(seconds: int) -> str: | |
| if seconds >= 86400: | |
| d = seconds // 86400 | |
| return f"{d}d" | |
| if seconds >= 3600: | |
| h = seconds // 3600 | |
| return f"{h}h" | |
| if seconds >= 60: | |
| m = seconds // 60 | |
| return f"{m}min" | |
| return f"{seconds}s" | |
| def build_domain_status_md( | |
| domain: str, | |
| datasets: list[str], | |
| root_dir: str = "results", | |
| ) -> str: | |
| """Build a compact markdown status line for a domain tab. | |
| Shows data source, last eval time, last evaluated time for this domain, | |
| and per-dataset refresh intervals. | |
| """ | |
| utc8 = timezone(timedelta(hours=8)) | |
| # 1. Load data source name from online_status.json | |
| status_path = Path(root_dir) / "online_status.json" | |
| data_source = "TS-Bench" | |
| if status_path.exists(): | |
| try: | |
| status = json.loads(status_path.read_text()) | |
| data_source = status.get("data_source", "TS-Bench") | |
| except Exception: | |
| pass | |
| # 2. Last evaluated time for this domain (max of all datasets in the domain) | |
| domain_latest_dt: datetime | None = None | |
| for ds in datasets: | |
| ds_base = ds.split("/")[0].strip() | |
| eval_dt = get_dataset_last_eval_time(ds_base, root_dir) | |
| if eval_dt: | |
| if domain_latest_dt is None or eval_dt > domain_latest_dt: | |
| domain_latest_dt = eval_dt | |
| last_evaluated = "n/a" | |
| if domain_latest_dt: | |
| last_evaluated = domain_latest_dt.astimezone(utc8).strftime("%Y-%m-%d %H:%M:%S") + " (UTC+8)" | |
| # 3. Auto-refresh per-dataset | |
| intervals = load_eval_intervals(root_dir) | |
| ds_parts: list[str] = [] | |
| for ds in datasets: | |
| short = dataset_short_name(ds) | |
| ds_base = ds.split("/")[0].strip() | |
| interval = intervals.get(ds_base) | |
| if interval: | |
| ds_parts.append(f"{short} every {_format_interval(interval)}") | |
| else: | |
| ds_parts.append(short) | |
| parts = [ | |
| f"**Data**: {data_source}", | |
| f"**Last eval**: {last_evaluated}", | |
| ] | |
| if ds_parts: | |
| parts.append("**Refresh**: " + " · ".join(ds_parts)) | |
| return " | ".join(parts) | |