File size: 15,141 Bytes
baee854
 
347de80
baee854
 
c29fb7f
 
 
 
 
 
 
 
 
df09e67
6ae0ddf
7204769
7e92145
8b323f8
6ae0ddf
8b323f8
347de80
 
 
0684be1
 
 
 
82e0e89
347de80
 
6ae0ddf
 
 
c29fb7f
dcc91c8
df09e67
0684be1
62f5d66
baee854
8b323f8
 
 
 
 
 
 
 
 
 
 
 
347de80
 
 
 
7204769
347de80
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1d3f6c0
 
8b323f8
347de80
8b323f8
7031160
82e0e89
 
 
 
 
 
0684be1
 
 
 
 
 
 
7031160
7e92145
0684be1
7e92145
 
 
8b323f8
0684be1
7e92145
0684be1
7e92145
7031160
34ca281
 
7031160
 
8b323f8
347de80
 
7e92145
df09e67
34ca281
df09e67
 
 
 
7204769
7031160
8b323f8
 
7e92145
8b323f8
c24f0a0
 
347de80
c24f0a0
0684be1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82e0e89
7e92145
82e0e89
 
 
 
 
 
c24f0a0
5edf8b0
c24f0a0
 
 
82e0e89
c24f0a0
 
5edf8b0
c24f0a0
 
82e0e89
c24f0a0
 
 
 
1d3f6c0
347de80
 
 
 
 
82e0e89
347de80
 
 
 
 
 
 
 
 
82e0e89
347de80
 
 
 
 
5edf8b0
347de80
 
 
82e0e89
347de80
 
 
0684be1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347de80
7e92145
8b323f8
 
7e92145
 
 
0684be1
7e92145
 
 
 
 
8b323f8
 
 
 
7031160
8b323f8
 
0684be1
8b323f8
 
 
 
 
 
 
 
 
 
 
0684be1
8b323f8
 
 
 
 
347de80
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8b323f8
baee854
d5f690f
 
 
 
 
 
 
 
 
 
 
df09e67
82e0e89
df09e67
 
347de80
 
 
0684be1
 
 
 
 
 
 
7e92145
8b323f8
 
df09e67
1d3f6c0
df09e67
 
d5f690f
dcc91c8
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
import json
import os
import re
from pathlib import Path

import gradio as gr

from src.about import (
    CITATION_BUTTON_LABEL,
    CITATION_BUTTON_TEXT,
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    TITLE,
)
from src.display.css_html_js import custom_css
from src.utils import (
    build_leaderboard_summary_html,
    build_domain_status_md,
    dataset_section_title,
    get_grouped_dfs,
    group_datasets_by_domain,
    load_baseline_rank_history,
    load_baseline_rank_history_weekly,
    load_forecast_snapshots,
    load_gift_aggregate_metadata_md,
    load_gift_aggregate_table,
    load_live_aggregate_metadata_md,
    load_live_aggregate_table,
    make_domain_pie_chart,
    make_forecast_plot,
    make_rank_trend_plot,
    prepare_ranks_table,
    prepare_values_table,
)

RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "space/results" if Path("space/results").exists() else "results")
REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
DOMAIN_OUTPUT_OFFSET = 13


def _dataset_tables(grouped: dict, datasets: list[str]) -> tuple[list, list]:
    values_tables = [
        prepare_values_table(grouped["dataset_values"].get(dataset, None))
        for dataset in datasets
    ]
    rank_tables = [
        prepare_ranks_table(grouped["dataset_ranks"].get(dataset, None))
        for dataset in datasets
    ]
    return values_tables, rank_tables


def _get_model_slugs(results_path: str) -> list[str]:
    """Return subdirs of results_path that contain a forecasts/ subdirectory."""
    slugs = []
    for p in sorted(Path(results_path).iterdir()):
        if p.is_dir() and (p / "forecasts").exists():
            slugs.append(p.name)
    return slugs


def _build_forecast_choices(results_path: str) -> dict[str, dict[str, dict]]:
    """Build {model_slug: {dataset_name: snapshot}} for all models."""
    choices: dict[str, dict[str, dict]] = {}
    for slug in _get_model_slugs(results_path):
        snaps = load_forecast_snapshots(results_path, slug)
        if snaps:
            choices[slug] = snaps
    return choices


def _short_dataset_key(dataset_name: str) -> str:
    """Extract the base dataset key from a leaderboard dataset name like 'ds/freq/term'."""
    return dataset_name.split("/")[0]


def refresh_leaderboard() -> tuple:
    grouped = get_grouped_dfs(RESULTS_PATH)
    datasets = grouped["datasets"]
    daily_df = load_baseline_rank_history(RESULTS_PATH)

    outputs: list = [
        make_domain_pie_chart(RESULTS_PATH),                # [0] domain pie chart
        prepare_values_table(grouped["overall_values"]),    # [1]
        prepare_ranks_table(grouped["overall_ranks"]),      # [2]
        daily_df,                                           # [3] daily table
        load_baseline_rank_history_weekly(RESULTS_PATH),    # [4] weekly table
        make_rank_trend_plot(daily_df),                     # [5] rank trend plot
        load_gift_aggregate_metadata_md(RESULTS_PATH),      # [6]
        load_gift_aggregate_table(RESULTS_PATH, "prediction_length"),  # [7]
        load_gift_aggregate_table(RESULTS_PATH, "domain"),             # [8]
        load_gift_aggregate_table(RESULTS_PATH, "frequency"),          # [9]
        load_live_aggregate_metadata_md(RESULTS_PATH),                 # [10]
        load_live_aggregate_table(RESULTS_PATH, "overall"),           # [11]
        load_live_aggregate_table(RESULTS_PATH, "rank"),              # [12]
    ]

    # [10 .. 10+D-1] per-domain status lines
    for domain, domain_datasets in DOMAIN_GROUPS.items():
        outputs.append(build_domain_status_md(domain, domain_datasets, RESULTS_PATH))

    value_tables, rank_tables = _dataset_tables(grouped, datasets)
    # [10+D .. 10+D+N-1] value tables
    outputs.extend(value_tables)
    # [10+D+N .. 10+D+2N-1] rank tables
    outputs.extend(rank_tables)
    return tuple(outputs)


grouped_initial = get_grouped_dfs(RESULTS_PATH)
DATASETS: list[str] = grouped_initial["datasets"]
DOMAIN_GROUPS = group_datasets_by_domain(DATASETS, RESULTS_PATH)
FORECAST_CHOICES = _build_forecast_choices(RESULTS_PATH)
MODEL_SLUGS = list(FORECAST_CHOICES.keys())
N_DOMAINS = len(DOMAIN_GROUPS)
INITIAL_OUTPUTS = refresh_leaderboard()

demo = gr.Blocks(css=custom_css)
with demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
    gr.HTML(build_leaderboard_summary_html(RESULTS_PATH))

    dataset_value_dfs: list[gr.Dataframe] = []
    dataset_rank_dfs: list[gr.Dataframe] = []
    domain_status_mds: list[gr.Markdown] = []
    dataset_idx = 0

    with gr.Tabs(elem_classes="tab-buttons"):
        # ── Overall Tab ──────────────────────────────────────────────────────
        with gr.TabItem("Overall"):
            live_aggregate_metadata_md = gr.Markdown(
                INITIAL_OUTPUTS[10],
                elem_classes="markdown-text",
            )
            gr.Markdown(
                "**Cumulative live evaluation.** MSE, RMSE, MAPE, CRPS, and Stability are lower-is-better; RTG is higher-is-better; more negative Improvement indicates a stronger decreasing-error trend.",
                elem_classes="markdown-text",
            )
            live_overall_df = gr.Dataframe(
                value=INITIAL_OUTPUTS[11],
                interactive=False,
                wrap=True,
                label="Up-to-now overall results",
            )
            live_rank_df = gr.Dataframe(
                value=INITIAL_OUTPUTS[12],
                interactive=False,
                wrap=True,
                label="Average Rank, Win Rate, and Elo",
            )
            gr.Markdown(
                "**Latest evaluation snapshot**",
                elem_classes="markdown-text",
            )
            gr.Markdown(
                "**Evaluated timestamps by domain** Β· hover for dataset names",
                elem_classes="markdown-text",
            )
            domain_pie_plot = gr.Plot(
                value=INITIAL_OUTPUTS[0],
                label="Domain distribution",
            )
            gr.Markdown(
                "Geometric mean of **absolute** metrics across all datasets. MSE/CRPS are lower-is-better; RankScore is higher-is-better.",
                elem_classes="markdown-text",
            )
            overall_values_df = gr.Dataframe(
                value=INITIAL_OUTPUTS[1],
                interactive=False,
                wrap=True,
                label="Metric values and RankScore",
            )
            overall_ranks_df = gr.Dataframe(
                value=INITIAL_OUTPUTS[2],
                interactive=False,
                wrap=True,
                label="Ranks (average per-dataset rank, lower is better)",
            )

            gr.Markdown(
                "**Baseline daily rank history** Β· average rank across all datasets per day (lower is better)",
                elem_classes="markdown-text",
            )
            baseline_history_df = gr.Dataframe(
                value=INITIAL_OUTPUTS[3],
                interactive=False,
                wrap=True,
                label="Baseline rank history β€” by day",
            )
            gr.Markdown(
                "**Baseline weekly rank history** Β· average of daily ranks per ISO week (lower is better)",
                elem_classes="markdown-text",
            )
            baseline_weekly_df = gr.Dataframe(
                value=INITIAL_OUTPUTS[4],
                interactive=False,
                wrap=True,
                label="Baseline rank history β€” by week",
            )
            gr.Markdown(
                "**Baseline rank trend** Β· MSE / CRPS / MASE / MAE rank over time",
                elem_classes="markdown-text",
            )
            rank_trend_plot = gr.Plot(
                value=INITIAL_OUTPUTS[5],
                label="Rank trend over time",
            )

        # ── GIFT-style grouped result tables ────────────────────────────────
        with gr.TabItem("GIFT-style Aggregates"):
            aggregate_metadata_md = gr.Markdown(
                INITIAL_OUTPUTS[6],
                elem_classes="markdown-text",
            )
            gr.Markdown(
                "MSE and CRPS are normalized per dataset configuration against Seasonal-Naive (1.0 = baseline). Rank is the mean per-configuration CRPS rank. Lower is better for all three metrics.",
                elem_classes="markdown-text",
            )
            with gr.Tabs():
                with gr.TabItem("Prediction Length"):
                    prediction_length_aggregate_df = gr.Dataframe(
                        value=INITIAL_OUTPUTS[7],
                        interactive=False,
                        wrap=True,
                        label="Results on TSFM_Bench aggregated by Prediction Length",
                    )
                with gr.TabItem("Domain"):
                    domain_aggregate_df = gr.Dataframe(
                        value=INITIAL_OUTPUTS[8],
                        interactive=False,
                        wrap=True,
                        label="Results on TSFM_Bench aggregated by Domain",
                    )
                with gr.TabItem("Frequency"):
                    frequency_aggregate_df = gr.Dataframe(
                        value=INITIAL_OUTPUTS[9],
                        interactive=False,
                        wrap=True,
                        label="Results on TSFM_Bench aggregated by Frequency",
                    )

        # ── Domain Tabs ───────────────────────────────────────────────────────
        domain_idx = 0
        for domain, domain_datasets in DOMAIN_GROUPS.items():
            with gr.TabItem(domain):
                # Per-domain status line (data source, last eval, refresh intervals)
                domain_status_mds.append(
                    gr.Markdown(
                        INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + domain_idx],
                        elem_classes="markdown-text",
                    )
                )
                domain_idx += 1

                for dataset in domain_datasets:
                    gr.Markdown(
                        dataset_section_title(dataset, RESULTS_PATH),
                        elem_classes="dataset-heading",
                    )
                    dataset_value_dfs.append(
                        gr.Dataframe(
                            value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + dataset_idx],
                            interactive=False,
                            wrap=True,
                            show_label=False,
                        )
                    )
                    gr.Markdown(
                        dataset_section_title(dataset, RESULTS_PATH, ranks=True),
                        elem_classes="dataset-heading",
                    )
                    dataset_rank_dfs.append(
                        gr.Dataframe(
                            value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + len(DATASETS) + dataset_idx],
                            interactive=False,
                            wrap=True,
                            show_label=False,
                        )
                    )

                    # ── Forecast plot for this dataset ──────────────────────
                    if MODEL_SLUGS:
                        ds_key = _short_dataset_key(dataset)
                        # Find which models have a snapshot for this dataset
                        available_models = [
                            slug for slug in MODEL_SLUGS
                            if any(
                                _short_dataset_key(k) == ds_key
                                for k in FORECAST_CHOICES.get(slug, {})
                            )
                        ]
                        if available_models:
                            gr.Markdown(
                                "**Forecast: true vs predicted** Β· select model",
                                elem_classes="dataset-heading",
                            )
                            model_dd = gr.Dropdown(
                                choices=available_models,
                                value=available_models[0],
                                label="Model",
                                interactive=True,
                            )

                            def _get_initial_snapshot(slug, dk=ds_key):
                                snaps = FORECAST_CHOICES.get(slug, {})
                                for name, snap in snaps.items():
                                    if _short_dataset_key(name) == dk:
                                        return snap
                                return None

                            forecast_plot = gr.Plot(
                                value=make_forecast_plot(_get_initial_snapshot(available_models[0])),
                                label="True vs Predicted",
                            )

                            def _make_update_fn(dk):
                                def _update(slug):
                                    snaps = FORECAST_CHOICES.get(slug, {})
                                    for name, snap in snaps.items():
                                        if _short_dataset_key(name) == dk:
                                            return make_forecast_plot(snap)
                                    return None
                                return _update

                            model_dd.change(
                                fn=_make_update_fn(ds_key),
                                inputs=[model_dd],
                                outputs=[forecast_plot],
                            )

                    dataset_idx += 1

    with gr.Accordion("About", open=False):
        gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

    with gr.Accordion("Citation", open=False):
        gr.Textbox(
            value=CITATION_BUTTON_TEXT,
            label=CITATION_BUTTON_LABEL,
            lines=12,
            show_copy_button=True,
        )

    refresh_outputs = [
        domain_pie_plot,
        overall_values_df,
        overall_ranks_df,
        baseline_history_df,
        baseline_weekly_df,
        rank_trend_plot,
        aggregate_metadata_md,
        prediction_length_aggregate_df,
        domain_aggregate_df,
        frequency_aggregate_df,
        live_aggregate_metadata_md,
        live_overall_df,
        live_rank_df,
        *domain_status_mds,
        *dataset_value_dfs,
        *dataset_rank_dfs,
    ]

    demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
    gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[], outputs=refresh_outputs)

if __name__ == "__main__":
    demo.queue(default_concurrency_limit=20).launch()