LiveHouse-TS / app.py
ziyuzhou02's picture
Deploy Space: update app.py
0684be1 verified
Raw History Blame
15.1 kB
import json
import os
import re
from pathlib import Path
import gradio as gr
from src.about import (
CITATION_BUTTON_LABEL,
CITATION_BUTTON_TEXT,
INTRODUCTION_TEXT,
LLM_BENCHMARKS_TEXT,
TITLE,
)
from src.display.css_html_js import custom_css
from src.utils import (
build_leaderboard_summary_html,
build_domain_status_md,
dataset_section_title,
get_grouped_dfs,
group_datasets_by_domain,
load_baseline_rank_history,
load_baseline_rank_history_weekly,
load_forecast_snapshots,
load_gift_aggregate_metadata_md,
load_gift_aggregate_table,
load_live_aggregate_metadata_md,
load_live_aggregate_table,
make_domain_pie_chart,
make_forecast_plot,
make_rank_trend_plot,
prepare_ranks_table,
prepare_values_table,
)
RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "space/results" if Path("space/results").exists() else "results")
REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
DOMAIN_OUTPUT_OFFSET = 13
def _dataset_tables(grouped: dict, datasets: list[str]) -> tuple[list, list]:
values_tables = [
prepare_values_table(grouped["dataset_values"].get(dataset, None))
for dataset in datasets
]
rank_tables = [
prepare_ranks_table(grouped["dataset_ranks"].get(dataset, None))
for dataset in datasets
]
return values_tables, rank_tables
def _get_model_slugs(results_path: str) -> list[str]:
"""Return subdirs of results_path that contain a forecasts/ subdirectory."""
slugs = []
for p in sorted(Path(results_path).iterdir()):
if p.is_dir() and (p / "forecasts").exists():
slugs.append(p.name)
return slugs
def _build_forecast_choices(results_path: str) -> dict[str, dict[str, dict]]:
"""Build {model_slug: {dataset_name: snapshot}} for all models."""
choices: dict[str, dict[str, dict]] = {}
for slug in _get_model_slugs(results_path):
snaps = load_forecast_snapshots(results_path, slug)
if snaps:
choices[slug] = snaps
return choices
def _short_dataset_key(dataset_name: str) -> str:
"""Extract the base dataset key from a leaderboard dataset name like 'ds/freq/term'."""
return dataset_name.split("/")[0]
def refresh_leaderboard() -> tuple:
grouped = get_grouped_dfs(RESULTS_PATH)
datasets = grouped["datasets"]
daily_df = load_baseline_rank_history(RESULTS_PATH)
outputs: list = [
make_domain_pie_chart(RESULTS_PATH), # [0] domain pie chart
prepare_values_table(grouped["overall_values"]), # [1]
prepare_ranks_table(grouped["overall_ranks"]), # [2]
daily_df, # [3] daily table
load_baseline_rank_history_weekly(RESULTS_PATH), # [4] weekly table
make_rank_trend_plot(daily_df), # [5] rank trend plot
load_gift_aggregate_metadata_md(RESULTS_PATH), # [6]
load_gift_aggregate_table(RESULTS_PATH, "prediction_length"), # [7]
load_gift_aggregate_table(RESULTS_PATH, "domain"), # [8]
load_gift_aggregate_table(RESULTS_PATH, "frequency"), # [9]
load_live_aggregate_metadata_md(RESULTS_PATH), # [10]
load_live_aggregate_table(RESULTS_PATH, "overall"), # [11]
load_live_aggregate_table(RESULTS_PATH, "rank"), # [12]
]
# [10 .. 10+D-1] per-domain status lines
for domain, domain_datasets in DOMAIN_GROUPS.items():
outputs.append(build_domain_status_md(domain, domain_datasets, RESULTS_PATH))
value_tables, rank_tables = _dataset_tables(grouped, datasets)
# [10+D .. 10+D+N-1] value tables
outputs.extend(value_tables)
# [10+D+N .. 10+D+2N-1] rank tables
outputs.extend(rank_tables)
return tuple(outputs)
grouped_initial = get_grouped_dfs(RESULTS_PATH)
DATASETS: list[str] = grouped_initial["datasets"]
DOMAIN_GROUPS = group_datasets_by_domain(DATASETS, RESULTS_PATH)
FORECAST_CHOICES = _build_forecast_choices(RESULTS_PATH)
MODEL_SLUGS = list(FORECAST_CHOICES.keys())
N_DOMAINS = len(DOMAIN_GROUPS)
INITIAL_OUTPUTS = refresh_leaderboard()
demo = gr.Blocks(css=custom_css)
with demo:
gr.HTML(TITLE)
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
gr.HTML(build_leaderboard_summary_html(RESULTS_PATH))
dataset_value_dfs: list[gr.Dataframe] = []
dataset_rank_dfs: list[gr.Dataframe] = []
domain_status_mds: list[gr.Markdown] = []
dataset_idx = 0
with gr.Tabs(elem_classes="tab-buttons"):
# ── Overall Tab ──────────────────────────────────────────────────────
with gr.TabItem("Overall"):
live_aggregate_metadata_md = gr.Markdown(
INITIAL_OUTPUTS[10],
elem_classes="markdown-text",
)
gr.Markdown(
"**Cumulative live evaluation.** MSE, RMSE, MAPE, CRPS, and Stability are lower-is-better; RTG is higher-is-better; more negative Improvement indicates a stronger decreasing-error trend.",
elem_classes="markdown-text",
)
live_overall_df = gr.Dataframe(
value=INITIAL_OUTPUTS[11],
interactive=False,
wrap=True,
label="Up-to-now overall results",
)
live_rank_df = gr.Dataframe(
value=INITIAL_OUTPUTS[12],
interactive=False,
wrap=True,
label="Average Rank, Win Rate, and Elo",
)
gr.Markdown(
"**Latest evaluation snapshot**",
elem_classes="markdown-text",
)
gr.Markdown(
"**Evaluated timestamps by domain** Β· hover for dataset names",
elem_classes="markdown-text",
)
domain_pie_plot = gr.Plot(
value=INITIAL_OUTPUTS[0],
label="Domain distribution",
)
gr.Markdown(
"Geometric mean of **absolute** metrics across all datasets. MSE/CRPS are lower-is-better; RankScore is higher-is-better.",
elem_classes="markdown-text",
)
overall_values_df = gr.Dataframe(
value=INITIAL_OUTPUTS[1],
interactive=False,
wrap=True,
label="Metric values and RankScore",
)
overall_ranks_df = gr.Dataframe(
value=INITIAL_OUTPUTS[2],
interactive=False,
wrap=True,
label="Ranks (average per-dataset rank, lower is better)",
)
gr.Markdown(
"**Baseline daily rank history** Β· average rank across all datasets per day (lower is better)",
elem_classes="markdown-text",
)
baseline_history_df = gr.Dataframe(
value=INITIAL_OUTPUTS[3],
interactive=False,
wrap=True,
label="Baseline rank history β€” by day",
)
gr.Markdown(
"**Baseline weekly rank history** Β· average of daily ranks per ISO week (lower is better)",
elem_classes="markdown-text",
)
baseline_weekly_df = gr.Dataframe(
value=INITIAL_OUTPUTS[4],
interactive=False,
wrap=True,
label="Baseline rank history β€” by week",
)
gr.Markdown(
"**Baseline rank trend** Β· MSE / CRPS / MASE / MAE rank over time",
elem_classes="markdown-text",
)
rank_trend_plot = gr.Plot(
value=INITIAL_OUTPUTS[5],
label="Rank trend over time",
)
# ── GIFT-style grouped result tables ────────────────────────────────
with gr.TabItem("GIFT-style Aggregates"):
aggregate_metadata_md = gr.Markdown(
INITIAL_OUTPUTS[6],
elem_classes="markdown-text",
)
gr.Markdown(
"MSE and CRPS are normalized per dataset configuration against Seasonal-Naive (1.0 = baseline). Rank is the mean per-configuration CRPS rank. Lower is better for all three metrics.",
elem_classes="markdown-text",
)
with gr.Tabs():
with gr.TabItem("Prediction Length"):
prediction_length_aggregate_df = gr.Dataframe(
value=INITIAL_OUTPUTS[7],
interactive=False,
wrap=True,
label="Results on TSFM_Bench aggregated by Prediction Length",
)
with gr.TabItem("Domain"):
domain_aggregate_df = gr.Dataframe(
value=INITIAL_OUTPUTS[8],
interactive=False,
wrap=True,
label="Results on TSFM_Bench aggregated by Domain",
)
with gr.TabItem("Frequency"):
frequency_aggregate_df = gr.Dataframe(
value=INITIAL_OUTPUTS[9],
interactive=False,
wrap=True,
label="Results on TSFM_Bench aggregated by Frequency",
)
# ── Domain Tabs ───────────────────────────────────────────────────────
domain_idx = 0
for domain, domain_datasets in DOMAIN_GROUPS.items():
with gr.TabItem(domain):
# Per-domain status line (data source, last eval, refresh intervals)
domain_status_mds.append(
gr.Markdown(
INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + domain_idx],
elem_classes="markdown-text",
)
)
domain_idx += 1
for dataset in domain_datasets:
gr.Markdown(
dataset_section_title(dataset, RESULTS_PATH),
elem_classes="dataset-heading",
)
dataset_value_dfs.append(
gr.Dataframe(
value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + dataset_idx],
interactive=False,
wrap=True,
show_label=False,
)
)
gr.Markdown(
dataset_section_title(dataset, RESULTS_PATH, ranks=True),
elem_classes="dataset-heading",
)
dataset_rank_dfs.append(
gr.Dataframe(
value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + len(DATASETS) + dataset_idx],
interactive=False,
wrap=True,
show_label=False,
)
)
# ── Forecast plot for this dataset ──────────────────────
if MODEL_SLUGS:
ds_key = _short_dataset_key(dataset)
# Find which models have a snapshot for this dataset
available_models = [
slug for slug in MODEL_SLUGS
if any(
_short_dataset_key(k) == ds_key
for k in FORECAST_CHOICES.get(slug, {})
)
]
if available_models:
gr.Markdown(
"**Forecast: true vs predicted** Β· select model",
elem_classes="dataset-heading",
)
model_dd = gr.Dropdown(
choices=available_models,
value=available_models[0],
label="Model",
interactive=True,
)
def _get_initial_snapshot(slug, dk=ds_key):
snaps = FORECAST_CHOICES.get(slug, {})
for name, snap in snaps.items():
if _short_dataset_key(name) == dk:
return snap
return None
forecast_plot = gr.Plot(
value=make_forecast_plot(_get_initial_snapshot(available_models[0])),
label="True vs Predicted",
)
def _make_update_fn(dk):
def _update(slug):
snaps = FORECAST_CHOICES.get(slug, {})
for name, snap in snaps.items():
if _short_dataset_key(name) == dk:
return make_forecast_plot(snap)
return None
return _update
model_dd.change(
fn=_make_update_fn(ds_key),
inputs=[model_dd],
outputs=[forecast_plot],
)
dataset_idx += 1
with gr.Accordion("About", open=False):
gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
with gr.Accordion("Citation", open=False):
gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
lines=12,
show_copy_button=True,
)
refresh_outputs = [
domain_pie_plot,
overall_values_df,
overall_ranks_df,
baseline_history_df,
baseline_weekly_df,
rank_trend_plot,
aggregate_metadata_md,
prediction_length_aggregate_df,
domain_aggregate_df,
frequency_aggregate_df,
live_aggregate_metadata_md,
live_overall_df,
live_rank_df,
*domain_status_mds,
*dataset_value_dfs,
*dataset_rank_dfs,
]
demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
if __name__ == "__main__":
demo.queue(default_concurrency_limit=20).launch()