import json import os import re from pathlib import Path import gradio as gr from src.about import ( CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE, ) from src.display.css_html_js import custom_css from src.utils import ( build_leaderboard_summary_html, build_domain_status_md, dataset_section_title, get_grouped_dfs, group_datasets_by_domain, load_baseline_rank_history, load_baseline_rank_history_weekly, load_forecast_snapshots, load_gift_aggregate_metadata_md, load_gift_aggregate_table, load_live_aggregate_metadata_md, load_live_aggregate_table, make_domain_pie_chart, make_forecast_plot, make_rank_trend_plot, prepare_ranks_table, prepare_values_table, ) RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "space/results" if Path("space/results").exists() else "results") REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300")) DOMAIN_OUTPUT_OFFSET = 13 def _dataset_tables(grouped: dict, datasets: list[str]) -> tuple[list, list]: values_tables = [ prepare_values_table(grouped["dataset_values"].get(dataset, None)) for dataset in datasets ] rank_tables = [ prepare_ranks_table(grouped["dataset_ranks"].get(dataset, None)) for dataset in datasets ] return values_tables, rank_tables def _get_model_slugs(results_path: str) -> list[str]: """Return subdirs of results_path that contain a forecasts/ subdirectory.""" slugs = [] for p in sorted(Path(results_path).iterdir()): if p.is_dir() and (p / "forecasts").exists(): slugs.append(p.name) return slugs def _build_forecast_choices(results_path: str) -> dict[str, dict[str, dict]]: """Build {model_slug: {dataset_name: snapshot}} for all models.""" choices: dict[str, dict[str, dict]] = {} for slug in _get_model_slugs(results_path): snaps = load_forecast_snapshots(results_path, slug) if snaps: choices[slug] = snaps return choices def _short_dataset_key(dataset_name: str) -> str: """Extract the base dataset key from a leaderboard dataset name like 'ds/freq/term'.""" return dataset_name.split("/")[0] def refresh_leaderboard() -> tuple: grouped = get_grouped_dfs(RESULTS_PATH) datasets = grouped["datasets"] daily_df = load_baseline_rank_history(RESULTS_PATH) outputs: list = [ make_domain_pie_chart(RESULTS_PATH), # [0] domain pie chart prepare_values_table(grouped["overall_values"]), # [1] prepare_ranks_table(grouped["overall_ranks"]), # [2] daily_df, # [3] daily table load_baseline_rank_history_weekly(RESULTS_PATH), # [4] weekly table make_rank_trend_plot(daily_df), # [5] rank trend plot load_gift_aggregate_metadata_md(RESULTS_PATH), # [6] load_gift_aggregate_table(RESULTS_PATH, "prediction_length"), # [7] load_gift_aggregate_table(RESULTS_PATH, "domain"), # [8] load_gift_aggregate_table(RESULTS_PATH, "frequency"), # [9] load_live_aggregate_metadata_md(RESULTS_PATH), # [10] load_live_aggregate_table(RESULTS_PATH, "overall"), # [11] load_live_aggregate_table(RESULTS_PATH, "rank"), # [12] ] # [10 .. 10+D-1] per-domain status lines for domain, domain_datasets in DOMAIN_GROUPS.items(): outputs.append(build_domain_status_md(domain, domain_datasets, RESULTS_PATH)) value_tables, rank_tables = _dataset_tables(grouped, datasets) # [10+D .. 10+D+N-1] value tables outputs.extend(value_tables) # [10+D+N .. 10+D+2N-1] rank tables outputs.extend(rank_tables) return tuple(outputs) grouped_initial = get_grouped_dfs(RESULTS_PATH) DATASETS: list[str] = grouped_initial["datasets"] DOMAIN_GROUPS = group_datasets_by_domain(DATASETS, RESULTS_PATH) FORECAST_CHOICES = _build_forecast_choices(RESULTS_PATH) MODEL_SLUGS = list(FORECAST_CHOICES.keys()) N_DOMAINS = len(DOMAIN_GROUPS) INITIAL_OUTPUTS = refresh_leaderboard() demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") gr.HTML(build_leaderboard_summary_html(RESULTS_PATH)) dataset_value_dfs: list[gr.Dataframe] = [] dataset_rank_dfs: list[gr.Dataframe] = [] domain_status_mds: list[gr.Markdown] = [] dataset_idx = 0 with gr.Tabs(elem_classes="tab-buttons"): # ── Overall Tab ────────────────────────────────────────────────────── with gr.TabItem("Overall"): live_aggregate_metadata_md = gr.Markdown( INITIAL_OUTPUTS[10], elem_classes="markdown-text", ) gr.Markdown( "**Cumulative live evaluation.** MSE, RMSE, MAPE, CRPS, and Stability are lower-is-better; RTG is higher-is-better; more negative Improvement indicates a stronger decreasing-error trend.", elem_classes="markdown-text", ) live_overall_df = gr.Dataframe( value=INITIAL_OUTPUTS[11], interactive=False, wrap=True, label="Up-to-now overall results", ) live_rank_df = gr.Dataframe( value=INITIAL_OUTPUTS[12], interactive=False, wrap=True, label="Average Rank, Win Rate, and Elo", ) gr.Markdown( "**Latest evaluation snapshot**", elem_classes="markdown-text", ) gr.Markdown( "**Evaluated timestamps by domain** · hover for dataset names", elem_classes="markdown-text", ) domain_pie_plot = gr.Plot( value=INITIAL_OUTPUTS[0], label="Domain distribution", ) gr.Markdown( "Geometric mean of **absolute** metrics across all datasets. MSE/CRPS are lower-is-better; RankScore is higher-is-better.", elem_classes="markdown-text", ) overall_values_df = gr.Dataframe( value=INITIAL_OUTPUTS[1], interactive=False, wrap=True, label="Metric values and RankScore", ) overall_ranks_df = gr.Dataframe( value=INITIAL_OUTPUTS[2], interactive=False, wrap=True, label="Ranks (average per-dataset rank, lower is better)", ) gr.Markdown( "**Baseline daily rank history** · average rank across all datasets per day (lower is better)", elem_classes="markdown-text", ) baseline_history_df = gr.Dataframe( value=INITIAL_OUTPUTS[3], interactive=False, wrap=True, label="Baseline rank history — by day", ) gr.Markdown( "**Baseline weekly rank history** · average of daily ranks per ISO week (lower is better)", elem_classes="markdown-text", ) baseline_weekly_df = gr.Dataframe( value=INITIAL_OUTPUTS[4], interactive=False, wrap=True, label="Baseline rank history — by week", ) gr.Markdown( "**Baseline rank trend** · MSE / CRPS / MASE / MAE rank over time", elem_classes="markdown-text", ) rank_trend_plot = gr.Plot( value=INITIAL_OUTPUTS[5], label="Rank trend over time", ) # ── GIFT-style grouped result tables ──────────────────────────────── with gr.TabItem("GIFT-style Aggregates"): aggregate_metadata_md = gr.Markdown( INITIAL_OUTPUTS[6], elem_classes="markdown-text", ) gr.Markdown( "MSE and CRPS are normalized per dataset configuration against Seasonal-Naive (1.0 = baseline). Rank is the mean per-configuration CRPS rank. Lower is better for all three metrics.", elem_classes="markdown-text", ) with gr.Tabs(): with gr.TabItem("Prediction Length"): prediction_length_aggregate_df = gr.Dataframe( value=INITIAL_OUTPUTS[7], interactive=False, wrap=True, label="Results on TSFM_Bench aggregated by Prediction Length", ) with gr.TabItem("Domain"): domain_aggregate_df = gr.Dataframe( value=INITIAL_OUTPUTS[8], interactive=False, wrap=True, label="Results on TSFM_Bench aggregated by Domain", ) with gr.TabItem("Frequency"): frequency_aggregate_df = gr.Dataframe( value=INITIAL_OUTPUTS[9], interactive=False, wrap=True, label="Results on TSFM_Bench aggregated by Frequency", ) # ── Domain Tabs ─────────────────────────────────────────────────────── domain_idx = 0 for domain, domain_datasets in DOMAIN_GROUPS.items(): with gr.TabItem(domain): # Per-domain status line (data source, last eval, refresh intervals) domain_status_mds.append( gr.Markdown( INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + domain_idx], elem_classes="markdown-text", ) ) domain_idx += 1 for dataset in domain_datasets: gr.Markdown( dataset_section_title(dataset, RESULTS_PATH), elem_classes="dataset-heading", ) dataset_value_dfs.append( gr.Dataframe( value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + dataset_idx], interactive=False, wrap=True, show_label=False, ) ) gr.Markdown( dataset_section_title(dataset, RESULTS_PATH, ranks=True), elem_classes="dataset-heading", ) dataset_rank_dfs.append( gr.Dataframe( value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + len(DATASETS) + dataset_idx], interactive=False, wrap=True, show_label=False, ) ) # ── Forecast plot for this dataset ────────────────────── if MODEL_SLUGS: ds_key = _short_dataset_key(dataset) # Find which models have a snapshot for this dataset available_models = [ slug for slug in MODEL_SLUGS if any( _short_dataset_key(k) == ds_key for k in FORECAST_CHOICES.get(slug, {}) ) ] if available_models: gr.Markdown( "**Forecast: true vs predicted** · select model", elem_classes="dataset-heading", ) model_dd = gr.Dropdown( choices=available_models, value=available_models[0], label="Model", interactive=True, ) def _get_initial_snapshot(slug, dk=ds_key): snaps = FORECAST_CHOICES.get(slug, {}) for name, snap in snaps.items(): if _short_dataset_key(name) == dk: return snap return None forecast_plot = gr.Plot( value=make_forecast_plot(_get_initial_snapshot(available_models[0])), label="True vs Predicted", ) def _make_update_fn(dk): def _update(slug): snaps = FORECAST_CHOICES.get(slug, {}) for name, snap in snaps.items(): if _short_dataset_key(name) == dk: return make_forecast_plot(snap) return None return _update model_dd.change( fn=_make_update_fn(ds_key), inputs=[model_dd], outputs=[forecast_plot], ) dataset_idx += 1 with gr.Accordion("About", open=False): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") with gr.Accordion("Citation", open=False): gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=12, show_copy_button=True, ) refresh_outputs = [ domain_pie_plot, overall_values_df, overall_ranks_df, baseline_history_df, baseline_weekly_df, rank_trend_plot, aggregate_metadata_md, prediction_length_aggregate_df, domain_aggregate_df, frequency_aggregate_df, live_aggregate_metadata_md, live_overall_df, live_rank_df, *domain_status_mds, *dataset_value_dfs, *dataset_rank_dfs, ] demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs) gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[], outputs=refresh_outputs) if __name__ == "__main__": demo.queue(default_concurrency_limit=20).launch()