LiveHouse-TS / src /utils.py
ziyuzhou02's picture
Deploy Space: update src
53c5879 verified
Raw History Blame
29.1 kB
from __future__ import annotations
import json
import os
import re
from datetime import datetime, timedelta, timezone
from pathlib import Path
import numpy as np
import pandas as pd
from scipy import stats
METRIC_COLUMNS = [
"eval_metrics/MSE[mean]",
"eval_metrics/MSE[0.5]",
"eval_metrics/MAE[0.5]",
"eval_metrics/MASE[0.5]",
"eval_metrics/MAPE[0.5]",
"eval_metrics/sMAPE[0.5]",
"eval_metrics/MSIS",
"eval_metrics/RMSE[mean]",
"eval_metrics/NRMSE[mean]",
"eval_metrics/ND[0.5]",
"eval_metrics/mean_weighted_sum_quantile_loss",
]
DISPLAY_METRICS = [
"eval_metrics/MASE[0.5]",
"eval_metrics/mean_weighted_sum_quantile_loss",
"eval_metrics/MAE[0.5]",
"eval_metrics/RMSE[mean]",
"eval_metrics/sMAPE[0.5]",
"eval_metrics/MSIS",
"eval_metrics/ND[0.5]",
"eval_metrics/NRMSE[mean]",
"eval_metrics/MAPE[0.5]",
]
METRIC_LABELS = {
"eval_metrics/MASE[0.5]": "MASE",
"eval_metrics/mean_weighted_sum_quantile_loss": "CRPS",
"eval_metrics/MAE[0.5]": "MAE",
"eval_metrics/RMSE[mean]": "RMSE",
"eval_metrics/sMAPE[0.5]": "sMAPE",
"eval_metrics/MSIS": "MSIS",
"eval_metrics/ND[0.5]": "ND",
"eval_metrics/NRMSE[mean]": "NRMSE",
"eval_metrics/MAPE[0.5]": "MAPE",
}
VALUE_COLUMNS = [METRIC_LABELS[m] for m in DISPLAY_METRICS]
RANK_COLUMNS = [f"{METRIC_LABELS[m]}_Rank" for m in DISPLAY_METRICS]
LEGACY_MODEL_NAMES = {
"TSFM1": "Chronos-Bolt-Tiny",
"TSFM2": "Chronos-Bolt-Base",
"TSFM3": "Chronos-2",
}
RESULT_COLUMNS = [
"dataset",
"model",
*METRIC_COLUMNS,
"domain",
"num_variates",
]
def format_number(value):
if isinstance(value, (int, float)):
if pd.isna(value):
return np.nan
if abs(value) >= 10**2:
return f"{value:.1e}"
return f"{value:.3f}"
return value
def aggregate_gmean(series: pd.Series) -> float:
values = series.dropna()
if values.empty:
return np.nan
return float(stats.gmean(values))
def aggregate_rank_mean(series: pd.Series) -> float:
return float(series.mean(skipna=True))
def format_df(df: pd.DataFrame) -> pd.DataFrame:
formatted = df.copy()
for col in formatted.columns:
if pd.api.types.is_numeric_dtype(formatted[col]):
formatted[col] = formatted[col].map(format_number)
formatted[col] = formatted[col].astype(float)
return formatted
def unify_freq(df: pd.DataFrame) -> pd.DataFrame:
df = df.copy()
df["frequency"] = df["frequency"].str.replace(r"\d+", "", regex=True)
df["frequency"] = df["frequency"].str.split("-").str[0]
return df
def load_all_results(root_dir: str = "results") -> pd.DataFrame:
frames = []
for subdir, _, files in os.walk(root_dir):
for file in files:
if file != "all_results.csv":
continue
frame = pd.read_csv(os.path.join(subdir, file))
if frame.empty:
continue
frames.append(frame)
if not frames:
return pd.DataFrame(columns=RESULT_COLUMNS)
df = pd.concat(frames, ignore_index=True)
df = df.dropna(subset=["dataset"], how="any")
df = df[df["dataset"].astype(str).str.len() > 0]
if df.empty:
return pd.DataFrame(columns=RESULT_COLUMNS)
df = df.sort_values(by=["model", "dataset"]).reset_index(drop=True)
df["model"] = df["model"].replace(LEGACY_MODEL_NAMES)
ALLOWED_MODELS = {
"Toto-2.0-1B",
"Moirai-1.1-Large",
"Chronos-2",
"Timer-S1",
"TiRex",
"TimesFM-2.5"
}
df = df[df["model"].isin(ALLOWED_MODELS)]
parts = df["dataset"].astype(str).str.split("/", expand=True)
if parts.shape[1] < 3:
for idx in range(parts.shape[1], 3):
parts[idx] = "short" if idx == 2 else ""
df["dataset"] = parts[0]
df["frequency"] = parts[1]
df["term_length"] = parts[2]
return df
def enrich_with_properties(df: pd.DataFrame, ds_properties_path: str) -> pd.DataFrame:
props = pd.read_csv(ds_properties_path)
props["dataset"] = props["dataset"].str.lower().str.replace(" ", "_")
props["dataset"] = props["dataset"].str.replace("-", "_")
props["dataset"] = props["dataset"].apply(lambda x: re.sub("_+", "_", x).strip("_"))
props_map = props.set_index("dataset").T.to_dict()
for dataset_key, values in props_map.items():
for key, value in values.items():
if key == "frequency" and df.loc[df["dataset"] == dataset_key, "frequency"].notna().all():
continue
df.loc[df["dataset"] == dataset_key, key] = value
return unify_freq(df)
def prepare_results_df(
root_dir: str = "results",
ds_properties: str | None = None,
) -> pd.DataFrame:
if ds_properties is None:
ds_properties = str(Path(root_dir) / "dataset_properties.csv")
df = load_all_results(root_dir)
if df.empty:
return df
if Path(ds_properties).exists():
df = enrich_with_properties(df, ds_properties)
for metric in METRIC_COLUMNS:
df[metric] = pd.to_numeric(df[metric], errors="coerce")
return _add_per_dataset_ranks(df)
def _add_per_dataset_ranks(df: pd.DataFrame) -> pd.DataFrame:
if df.empty:
return df
df = df.copy()
group_keys = ["dataset", "term_length", "frequency"]
for metric in DISPLAY_METRICS:
rank_col = f"Rank_{metric}"
df[rank_col] = df.groupby(group_keys)[metric].rank(method="first", ascending=True)
return df
def _empty_grouped_dfs() -> dict[str, pd.DataFrame | list[str]]:
rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS]
return {
"overall_values": pd.DataFrame(columns=DISPLAY_METRICS),
"overall_ranks": pd.DataFrame(columns=rank_cols),
"dataset_values": {},
"dataset_ranks": {},
"datasets": [],
}
def get_grouped_dfs(
root_dir: str = "results",
ds_properties: str | None = None,
) -> dict[str, pd.DataFrame | list[str]]:
df = prepare_results_df(root_dir, ds_properties)
if df.empty:
return _empty_grouped_dfs()
rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS]
overall_values = df.groupby(["model"])[DISPLAY_METRICS].agg(aggregate_gmean)
overall_ranks = df.groupby(["model"])[rank_cols].agg(aggregate_rank_mean)
overall_values = overall_values.rename(columns=METRIC_LABELS)
overall_ranks = overall_ranks.rename(
columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS}
)
datasets = sorted(df["dataset"].dropna().unique())
dataset_values: dict[str, pd.DataFrame] = {}
dataset_ranks: dict[str, pd.DataFrame] = {}
for dataset in datasets:
subset = df[df["dataset"] == dataset]
if subset.empty:
continue
values = subset.set_index("model")[DISPLAY_METRICS].rename(columns=METRIC_LABELS)
ranks = subset.set_index("model")[rank_cols].rename(
columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS}
)
dataset_values[dataset] = values
dataset_ranks[dataset] = ranks
return {
"overall_values": overall_values,
"overall_ranks": overall_ranks,
"dataset_values": dataset_values,
"dataset_ranks": dataset_ranks,
"datasets": datasets,
}
def prepare_values_table(values_df: pd.DataFrame) -> pd.DataFrame:
if values_df is None or values_df.empty:
return pd.DataFrame(columns=["Model"] + VALUE_COLUMNS)
table = values_df.reset_index().rename(columns={"model": "Model"})
table = format_df(table)
cols = ["Model"] + [col for col in VALUE_COLUMNS if col in table.columns]
table = table[cols]
if "MASE" in table.columns and not table["MASE"].isna().all():
return table.sort_values(by=["MASE"])
return table
def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame:
if ranks_df is None or ranks_df.empty:
return pd.DataFrame(columns=["Model"] + RANK_COLUMNS)
table = ranks_df.reset_index().rename(columns={"model": "Model"})
table = format_df(table)
cols = ["Model"] + [col for col in RANK_COLUMNS if col in table.columns]
table = table[cols]
if "MASE_Rank" in table.columns and not table["MASE_Rank"].isna().all():
return table.sort_values(by=["MASE_Rank"])
return table
DOMAIN_DISPLAY = {
"Weather": "Climate",
"Air Quality": "Climate",
"Transport": "Transport",
"Finance": "Finance",
"Ocean": "Ocean",
"Hydrology": "Hydrology",
"Hazards": "Hazards",
"Web": "Web",
}
DATASET_SHORT_NAMES = {
"binance_btcusdt_1h_close": "BTC/USDT",
"coingecko_bitcoin_7d_price_usd": "Bitcoin USD",
"nasa_power_shanghai_hourly_t2m": "Shanghai temperature (NASA POWER)",
"nasa_power_shanghai_hourly_T2M": "Shanghai temperature (NASA POWER)",
"open_meteo_shanghai_hourly_temperature_2m": "Shanghai temperature",
"open_meteo_shanghai_air_quality_hourly_pm2_5": "Shanghai PM2.5",
"noaa_coops_sf_water_level_water_level": "SF Bay water level",
"noaa_ndbc_46013_realtime_wave_height": "Ocean wave height",
"nws_ksfo_observations_temperature": "KSFO temperature",
"usgs_earthquake_all_week_hourly_earthquake_count": "Earthquake count",
"usgs_potomac_iv_usgs_00060": "Potomac river flow",
"wikimedia_time_series_hourly_pageviews": "Wikipedia pageviews",
}
def display_domain(raw_domain: str) -> str:
return DOMAIN_DISPLAY.get(raw_domain, raw_domain)
def dataset_short_name(dataset: str) -> str:
base = dataset.split("/")[0].strip()
if base in DATASET_SHORT_NAMES:
return DATASET_SHORT_NAMES[base]
label = base.replace("_", " ")
for prefix in ("open meteo ", "noaa coops ", "noaa ndbc ", "nws ", "usgs ", "wikimedia "):
if label.lower().startswith(prefix):
label = label[len(prefix) :]
return label[:40] + ("…" if len(label) > 40 else "")
def lookup_dataset_domain(dataset: str, root_dir: str = "results") -> str:
# Primary: dataset_properties.csv
base = dataset.split("/")[0].strip()
props_path = Path(root_dir) / "dataset_properties.csv"
if props_path.exists():
props = pd.read_csv(props_path)
match = props[props["dataset"].str.lower() == base.lower()]
if not match.empty and "domain" in match.columns:
return str(match.iloc[0]["domain"])
# Fallback: read domain column from any all_results.csv
for subdir in Path(root_dir).iterdir():
csv_path = subdir / "all_results.csv"
if not csv_path.exists():
continue
try:
df = pd.read_csv(csv_path)
if "domain" not in df.columns or "dataset" not in df.columns:
continue
# dataset column in all_results is "dataset_id/freq/term", strip suffix
df["_ds_key"] = df["dataset"].astype(str).str.split("/").str[0]
match = df[df["_ds_key"].str.lower() == base.lower()]
if not match.empty:
return str(match.iloc[0]["domain"])
except Exception:
continue
return "Other"
DOMAIN_TAB_ORDER = [
"Finance",
"Climate",
"Ocean",
"Hydrology",
"Transport",
"Hazards",
"Web",
"Other",
]
def group_datasets_by_domain(
datasets: list[str],
root_dir: str = "results",
) -> dict[str, list[str]]:
by_domain: dict[str, list[str]] = {}
for dataset in datasets:
domain = display_domain(lookup_dataset_domain(dataset, root_dir))
by_domain.setdefault(domain, []).append(dataset)
ordered: dict[str, list[str]] = {}
for domain in DOMAIN_TAB_ORDER:
if domain in by_domain:
ordered[domain] = by_domain.pop(domain)
for domain in sorted(by_domain):
ordered[domain] = by_domain[domain]
return ordered
def dataset_tab_label(dataset: str, root_dir: str = "results") -> str:
raw_domain = lookup_dataset_domain(dataset, root_dir)
domain = display_domain(raw_domain)
short = dataset_short_name(dataset)
return f"{domain} · {short}"
def dataset_section_title(dataset: str, root_dir: str, *, ranks: bool = False) -> str:
label = dataset_tab_label(dataset, root_dir)
if ranks:
return f"**{label} · ranks (lower is better)**"
return f"**{label} · metric values (absolute, lower is better)**"
BASELINE_RANK_HISTORY_COLUMNS = [
"date",
"model",
"MASE_Rank",
"CRPS_Rank",
"MAE_Rank",
"RMSE_Rank",
"sMAPE_Rank",
"MSIS_Rank",
"ND_Rank",
"NRMSE_Rank",
"MAPE_Rank",
]
_RANK_COLS = [c for c in BASELINE_RANK_HISTORY_COLUMNS if c.endswith("_Rank")]
_DAILY_DISPLAY_COLS = ["date"] + _RANK_COLS
_WEEKLY_DISPLAY_COLS = ["week"] + _RANK_COLS
def _read_rank_history(path: Path, date_col: str) -> pd.DataFrame:
"""Read a rank history CSV, coerce rank columns to float, sort by date_col."""
if not path.exists():
return pd.DataFrame(columns=[date_col] + _RANK_COLS)
try:
df = pd.read_csv(path)
except Exception:
return pd.DataFrame(columns=[date_col] + _RANK_COLS)
for col in _RANK_COLS:
if col in df.columns:
df[col] = pd.to_numeric(df[col], errors="coerce").round(2)
if date_col in df.columns:
df = df.sort_values(date_col)
present = [c for c in [date_col] + _RANK_COLS if c in df.columns]
return df[present].reset_index(drop=True)
def load_baseline_rank_history(root_dir: str = "results") -> pd.DataFrame:
"""Load daily baseline rank history (baseline_rank_history_daily.csv)."""
# Also try the legacy single-file name for backward-compat
for name in ("baseline_rank_history_daily.csv", "baseline_rank_history.csv"):
path = Path(root_dir) / name
if path.exists():
return _read_rank_history(path, "date")
return pd.DataFrame(columns=_DAILY_DISPLAY_COLS)
def load_baseline_rank_history_weekly(root_dir: str = "results") -> pd.DataFrame:
"""Load weekly aggregated baseline rank history (baseline_rank_history_weekly.csv)."""
return _read_rank_history(Path(root_dir) / "baseline_rank_history_weekly.csv", "week")
# ---------------------------------------------------------------------------
# Forecast snapshot utilities
# ---------------------------------------------------------------------------
def load_forecast_snapshots(root_dir: str, model_slug: str) -> dict[str, dict]:
"""Load all forecast JSON files for a given model slug.
Returns a dict keyed by the dataset display name (from snapshot["dataset"]).
"""
forecasts_dir = Path(root_dir) / model_slug / "forecasts"
snapshots: dict[str, dict] = {}
if not forecasts_dir.exists():
return snapshots
for fp in sorted(forecasts_dir.glob("*.json")):
try:
snap = json.loads(fp.read_text())
snapshots[snap.get("dataset", fp.stem)] = snap
except Exception:
pass
return snapshots
def _get_plotly():
"""Lazy import plotly to avoid import errors in non-chart contexts."""
try:
import plotly.graph_objects as go
return go
except ImportError:
return None
def make_forecast_plot(snapshot: dict | None) -> object | None:
"""Build a plotly Figure showing context + actuals vs p50 forecast with p10/p90 band.
Returns None if snapshot is None or plotly is unavailable.
"""
go = _get_plotly()
if go is None or snapshot is None:
return None
ctx = snapshot.get("context", [])
actuals = snapshot.get("actuals", [])
p50 = snapshot.get("p50", [])
p10 = snapshot.get("p10", [])
p90 = snapshot.get("p90", [])
n_ctx = len(ctx)
n_fut = len(actuals)
timestamps = snapshot.get("timestamps", [])
if timestamps and len(timestamps) == n_ctx + n_fut:
ctx_x = timestamps[:n_ctx]
fut_x = timestamps[n_ctx:]
x_title = "Time"
vline_x = ctx_x[-1] if n_ctx > 0 else fut_x[0]
else:
ctx_x = list(range(n_ctx))
fut_x = list(range(n_ctx, n_ctx + n_fut))
x_title = "Time step"
vline_x = n_ctx - 1.0 if n_ctx > 0 else 0
if n_ctx > 0:
fut_x = [ctx_x[-1]] + list(fut_x)
actuals = [ctx[-1]] + list(actuals)
p50 = [ctx[-1]] + list(p50) if p50 else []
p10 = [ctx[-1]] + list(p10) if p10 else []
p90 = [ctx[-1]] + list(p90) if p90 else []
n_fut = len(actuals)
fig = go.Figure()
# Context (historical)
fig.add_trace(go.Scatter(
x=ctx_x, y=ctx,
mode="lines",
name="Context (history)",
line=dict(color="#94a3b8", width=1.5),
))
# Actuals (ground truth for forecast horizon)
fig.add_trace(go.Scatter(
x=fut_x, y=actuals,
mode="lines",
name="Actual",
line=dict(color="#3b82f6", width=2),
))
# p10-p90 confidence band
if p10 and p90 and len(p10) == n_fut and len(p90) == n_fut:
fig.add_trace(go.Scatter(
x=fut_x + fut_x[::-1],
y=p90 + p10[::-1],
fill="toself",
fillcolor="rgba(239,68,68,0.15)",
line=dict(color="rgba(255,255,255,0)"),
name="p10-p90 band",
showlegend=True,
))
# p50 forecast
if p50 and len(p50) == n_fut:
fig.add_trace(go.Scatter(
x=fut_x, y=p50,
mode="lines",
name="Forecast (p50)",
line=dict(color="#ef4444", width=2, dash="dash"),
))
dataset_label = snapshot.get("dataset", "").split("/")[0]
model_label = snapshot.get("model", "")
evaluated_at = snapshot.get("evaluated_at", "")[:10]
fig.update_layout(
title=dict(
text=f"{dataset_label} — {model_label} ({evaluated_at})",
font=dict(size=14),
),
xaxis_title=x_title,
yaxis_title="Value",
legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1),
margin=dict(l=40, r=20, t=60, b=40),
height=350,
paper_bgcolor="rgba(0,0,0,0)",
plot_bgcolor="rgba(0,0,0,0)",
font=dict(color="black"),
xaxis=dict(gridcolor="rgba(0,0,0,0.08)"),
yaxis=dict(gridcolor="rgba(0,0,0,0.08)"),
)
# Vertical line separating context from forecast
fig.add_vline(
x=vline_x,
line_dash="dot",
line_color="rgba(0,0,0,0.3)",
annotation_text="forecast start",
annotation_font_color="black",
)
return fig
def make_rank_trend_plot(history_df: pd.DataFrame, metrics: list[str] | None = None) -> object | None:
"""Build a plotly Figure showing selected rank metrics over time (date on X axis).
Args:
history_df: DataFrame from load_baseline_rank_history() with a 'date' column.
metrics: List of rank column names to plot. Defaults to MASE_Rank and CRPS_Rank.
Returns None if fewer than 1 data point or plotly unavailable.
"""
go = _get_plotly()
if go is None or history_df is None or history_df.empty:
return None
if metrics is None:
metrics = ["MASE_Rank", "CRPS_Rank", "MAE_Rank", "RMSE_Rank"]
date_col = "date" if "date" in history_df.columns else "week"
present_metrics = [m for m in metrics if m in history_df.columns]
if not present_metrics:
return None
COLORS = ["#f97316", "#a78bfa", "#34d399", "#60a5fa", "#fb7185", "#fbbf24"]
fig = go.Figure()
for i, col in enumerate(present_metrics):
fig.add_trace(go.Scatter(
x=history_df[date_col],
y=history_df[col],
mode="lines+markers",
name=col.replace("_Rank", ""),
line=dict(color=COLORS[i % len(COLORS)], width=2),
marker=dict(size=6),
))
fig.update_layout(
title=dict(text="Baseline rank trend over time (lower is better)", font=dict(size=14)),
xaxis_title=date_col.capitalize(),
yaxis_title="Average rank",
legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1),
margin=dict(l=40, r=20, t=60, b=40),
height=320,
paper_bgcolor="rgba(0,0,0,0)",
plot_bgcolor="rgba(0,0,0,0)",
font=dict(color="black"),
xaxis=dict(gridcolor="rgba(0,0,0,0.08)"),
yaxis=dict(gridcolor="rgba(0,0,0,0.08)"),
)
return fig
DOMAIN_COLORS = {
"Finance": "#f97316",
"Climate": "#60a5fa",
"Ocean": "#34d399",
"Hydrology": "#818cf8",
"Transport": "#fbbf24",
"Hazards": "#fb7185",
"Web": "#a78bfa",
"Other": "#94a3b8",
}
def _today_start_utc8() -> datetime:
"""Return midnight today in UTC+8 as an aware datetime."""
utc8 = timezone(timedelta(hours=8))
return datetime.now(utc8).replace(hour=0, minute=0, second=0, microsecond=0)
def get_dataset_last_eval_time(ds_base: str, root_dir: str = "results") -> datetime | None:
latest_dt: datetime | None = None
ALLOWED_SLUGS = {"chronos_2", "moirai_1_1_large", "timer_s1", "timesfm_2_5", "tirex", "toto_2_0_1b"}
# Scan model directories
for model_dir in Path(root_dir).iterdir():
if not model_dir.is_dir() or model_dir.name not in ALLOWED_SLUGS:
continue
forecasts_dir = model_dir / "forecasts"
if not forecasts_dir.is_dir():
continue
# Find files matching the base name
for fp in forecasts_dir.glob(f"{ds_base}*.json"):
try:
# To be super fast, regex search for evaluated_at
content = fp.read_text()
match = re.search(r'"evaluated_at"\s*:\s*"([^"]+)"', content)
if match:
eval_at = match.group(1)
dt = datetime.fromisoformat(eval_at.replace("Z", "+00:00"))
if latest_dt is None or dt > latest_dt:
latest_dt = dt
except Exception:
continue
return latest_dt
def make_domain_pie_chart(root_dir: str = "results") -> object | None:
"""Build a plotly Pie chart showing evaluated timestamp count per domain today.
Reads eval_history.jsonl to count total evaluated timestamps since UTC+8 00:00 today,
grouped by domain.
"""
go = _get_plotly()
if go is None:
return None
# Domain mapping from dataset_properties.csv
props_path = Path(root_dir) / "dataset_properties.csv"
if not props_path.exists():
return None
try:
props = pd.read_csv(props_path)
except Exception:
return None
domain_map: dict[str, str] = {}
for _, row in props.iterrows():
domain_map[str(row["dataset"]).lower()] = display_domain(str(row["domain"]))
def get_domain(ds_name: str) -> str:
ds_lower = ds_name.lower()
if ds_lower not in domain_map:
domain_map[ds_lower] = display_domain(lookup_dataset_domain(ds_name, root_dir))
return domain_map[ds_lower]
today_start = _today_start_utc8()
utc8 = today_start.tzinfo
# Read eval_history.jsonl (Cumulative)
history_path = Path(root_dir) / "eval_history.jsonl"
timestamp_counts: dict[str, int] = {} # domain → total timestamps
domain_datasets: dict[str, set[str]] = {} # domain → set of dataset short names
if history_path.exists():
try:
with open(history_path) as f:
for line in f:
if not line.strip():
continue
entry = json.loads(line)
ds_base = entry.get("dataset", "")
pred_len = entry.get("prediction_length", 0)
domain = get_domain(ds_base)
timestamp_counts[domain] = timestamp_counts.get(domain, 0) + pred_len
domain_datasets.setdefault(domain, set()).add(dataset_short_name(ds_base))
except Exception:
pass
# Fallback to cumulative snapshots count (no today filter) if empty
if not timestamp_counts:
seen_datasets: set[str] = set()
ALLOWED_SLUGS = {"chronos_2", "moirai_1_1_large", "timer_s1", "timesfm_2_5", "tirex", "toto_2_0_1b"}
for model_dir in sorted(Path(root_dir).iterdir()):
if not model_dir.is_dir() or model_dir.name not in ALLOWED_SLUGS:
continue
forecasts_dir = model_dir / "forecasts"
if not forecasts_dir.is_dir():
continue
for fp in forecasts_dir.glob("*.json"):
try:
snap = json.loads(fp.read_text())
except Exception:
continue
ds_full = snap.get("dataset", "")
ds_base = ds_full.split("/")[0]
if ds_base in seen_datasets:
continue
seen_datasets.add(ds_base)
domain = get_domain(ds_base)
n_ts = len(snap.get("context", [])) + len(snap.get("actuals", []))
timestamp_counts[domain] = timestamp_counts.get(domain, 0) + n_ts
domain_datasets.setdefault(domain, set()).add(dataset_short_name(ds_base))
if not timestamp_counts:
return None
labels = list(timestamp_counts.keys())
values = list(timestamp_counts.values())
hover = [", ".join(sorted(domain_datasets.get(d, []))) for d in labels]
colors = [DOMAIN_COLORS.get(label, "#94a3b8") for label in labels]
fig = go.Figure(data=[go.Pie(
labels=labels,
values=values,
hovertext=hover,
hoverinfo="label+percent+text",
textinfo="label+value+percent",
textposition="inside",
textfont=dict(size=14, color="white"),
insidetextorientation="radial",
marker=dict(colors=colors),
hole=0.25,
)])
fig.update_layout(
title=dict(
text="Evaluated timestamps by domain",
font=dict(size=16, weight="bold"),
),
margin=dict(l=20, r=20, t=60, b=20),
height=500,
paper_bgcolor="rgba(0,0,0,0)",
plot_bgcolor="rgba(0,0,0,0)",
font=dict(color="black"),
showlegend=True,
legend=dict(
orientation="h",
yanchor="bottom",
y=-0.1,
xanchor="center",
x=0.5
)
)
return fig
# ---------------------------------------------------------------------------
# Per-domain status helpers
# ---------------------------------------------------------------------------
def load_eval_intervals(root_dir: str = "results") -> dict[str, int]:
"""Load per-dataset eval intervals from dataset_eval_intervals.json."""
path = Path(root_dir) / "dataset_eval_intervals.json"
if not path.exists():
return {}
try:
return {str(k): int(v) for k, v in json.loads(path.read_text()).items()}
except Exception:
return {}
def _format_interval(seconds: int) -> str:
if seconds >= 86400:
d = seconds // 86400
return f"{d}d"
if seconds >= 3600:
h = seconds // 3600
return f"{h}h"
if seconds >= 60:
m = seconds // 60
return f"{m}min"
return f"{seconds}s"
def build_domain_status_md(
domain: str,
datasets: list[str],
root_dir: str = "results",
) -> str:
"""Build a compact markdown status line for a domain tab.
Shows data source, last eval time, last evaluated time for this domain,
and per-dataset refresh intervals.
"""
utc8 = timezone(timedelta(hours=8))
# 1. Load data source name from online_status.json
status_path = Path(root_dir) / "online_status.json"
data_source = "TS-Bench"
if status_path.exists():
try:
status = json.loads(status_path.read_text())
data_source = status.get("data_source", "TS-Bench")
except Exception:
pass
# 2. Last evaluated time for this domain (max of all datasets in the domain)
domain_latest_dt: datetime | None = None
for ds in datasets:
ds_base = ds.split("/")[0].strip()
eval_dt = get_dataset_last_eval_time(ds_base, root_dir)
if eval_dt:
if domain_latest_dt is None or eval_dt > domain_latest_dt:
domain_latest_dt = eval_dt
last_evaluated = "n/a"
if domain_latest_dt:
last_evaluated = domain_latest_dt.astimezone(utc8).strftime("%Y-%m-%d %H:%M:%S") + " (UTC+8)"
# 3. Auto-refresh per-dataset
intervals = load_eval_intervals(root_dir)
ds_parts: list[str] = []
for ds in datasets:
short = dataset_short_name(ds)
ds_base = ds.split("/")[0].strip()
interval = intervals.get(ds_base)
if interval:
ds_parts.append(f"{short} every {_format_interval(interval)}")
else:
ds_parts.append(short)
parts = [
f"**Data**: {data_source}",
f"**Last eval**: {last_evaluated}",
]
if ds_parts:
parts.append("**Refresh**: " + " · ".join(ds_parts))
return " | ".join(parts)