ziyuzhou02 commited on
Commit
f948216
·
verified ·
1 Parent(s): 6ac88d4

Deploy Space: update app.py

Browse files
Files changed (1) hide show
  1. app.py +70 -144
app.py CHANGED
@@ -24,80 +24,29 @@ from src.utils import (
24
  dataset_section_title,
25
  get_grouped_dfs,
26
  group_datasets_by_domain,
27
- load_baseline_rank_history,
28
- load_fixed_window_rank,
29
- live_rank_history_model_choices,
30
  load_gift_aggregate_metadata_md,
31
  load_gift_aggregate_table,
32
- load_live_aggregate_metadata_md,
33
- load_live_aggregate_table,
34
  load_forecast_snapshots,
35
  make_domain_pie_chart,
36
  make_forecast_plot,
37
- make_rank_trend_plot,
38
  prepare_ranks_table,
39
  prepare_values_table,
40
  )
41
 
42
  RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "space/results" if Path("space/results").exists() else "results")
43
  REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
44
- DOMAIN_OUTPUT_OFFSET = 15
45
  MODEL_COLUMN_WIDTH = "180px"
46
  RANK_MODEL_COLUMN_WIDTH = MODEL_COLUMN_WIDTH
47
  RANK_INDEX_COLUMN_WIDTH = MODEL_COLUMN_WIDTH
48
  METRIC_COLUMN_WIDTH = "118px"
49
  MODEL_COLUMN_NAMES = {"model"}
50
 
51
- RANK_HISTORY_ALL_MODELS = "All tracked models"
52
-
53
-
54
- def _rank_history_model_choices(daily_df=None) -> list[str]:
55
- models = live_rank_history_model_choices(RESULTS_PATH)
56
- if not models:
57
- if daily_df is None:
58
- daily_df = load_baseline_rank_history(RESULTS_PATH)
59
- if daily_df is not None and not daily_df.empty and "model" in daily_df.columns:
60
- models = sorted(
61
- {str(model).strip() for model in daily_df["model"].dropna() if str(model).strip()}
62
- )
63
- return [RANK_HISTORY_ALL_MODELS] + models
64
-
65
-
66
- def _default_rank_history_model(daily_df=None) -> str:
67
- if daily_df is None:
68
- daily_df = load_baseline_rank_history(RESULTS_PATH)
69
- if daily_df is None or daily_df.empty or "model" not in daily_df.columns:
70
- return RANK_HISTORY_ALL_MODELS
71
- latest = daily_df.dropna(subset=["model"])
72
- if latest.empty:
73
- return RANK_HISTORY_ALL_MODELS
74
- if "date" in latest.columns:
75
- latest = latest.sort_values("date")
76
- model = str(latest.iloc[-1]["model"]).strip()
77
- return model or RANK_HISTORY_ALL_MODELS
78
-
79
-
80
- def _filter_rank_history_table(table, selected_model: str | None):
81
- if table is None or getattr(table, "empty", True):
82
- return table
83
- selected = (selected_model or "").strip()
84
- if not selected or selected == RANK_HISTORY_ALL_MODELS or "model" not in table.columns:
85
- return table.reset_index(drop=True)
86
- return table[table["model"].astype(str) == selected].reset_index(drop=True)
87
-
88
-
89
- def _rank_history_outputs(selected_model: str | None = None):
90
- daily = load_baseline_rank_history(RESULTS_PATH)
91
- selected = selected_model or _default_rank_history_model(daily)
92
- one_day = _filter_rank_history_table(load_fixed_window_rank(RESULTS_PATH, "1d"), selected)
93
- seven_days = _filter_rank_history_table(load_fixed_window_rank(RESULTS_PATH, "7d"), selected)
94
- thirty_days = _filter_rank_history_table(load_fixed_window_rank(RESULTS_PATH, "30d"), selected)
95
- trend = _filter_rank_history_table(daily, selected)
96
  return (
97
- _rank_table_html(one_day),
98
- _rank_table_html(seven_days),
99
- _rank_table_html(thirty_days),
100
- make_rank_trend_plot(trend),
101
  )
102
 
103
 
@@ -306,37 +255,38 @@ def _make_forecast_plotter(dataset_key: str):
306
  return update_forecast_plot
307
 
308
 
309
- def refresh_leaderboard(selected_rank_model: str | None = None) -> tuple:
310
  grouped = get_grouped_dfs(RESULTS_PATH)
311
  datasets = grouped["datasets"]
312
- rank_history_daily, rank_history_weekly, rank_history_monthly, rank_history_plot = _rank_history_outputs(selected_rank_model)
 
 
313
 
314
  outputs: list = [
315
  build_leaderboard_summary_html(RESULTS_PATH), # [0]
316
- make_domain_pie_chart(RESULTS_PATH), # [1] domain pie chart
317
- prepare_values_table(grouped["overall_values"]), # [2]
318
- _rank_table_html(prepare_ranks_table(grouped["overall_ranks"])), # [3]
319
- rank_history_daily, # [4] selected-model daily table
320
- rank_history_weekly, # [5] selected-model weekly table
321
- rank_history_monthly, # [6] selected-model monthly table
322
- rank_history_plot, # [7] selected-model rank trend plot
323
- load_gift_aggregate_metadata_md(RESULTS_PATH), # [8]
324
- load_gift_aggregate_table(RESULTS_PATH, "prediction_length"), # [9]
325
- load_gift_aggregate_table(RESULTS_PATH, "domain"), # [10]
326
- load_gift_aggregate_table(RESULTS_PATH, "frequency"), # [11]
327
- load_live_aggregate_metadata_md(RESULTS_PATH), # [12]
328
- load_live_aggregate_table(RESULTS_PATH, "overall"), # [13]
329
- _rank_table_html(load_live_aggregate_table(RESULTS_PATH, "rank")), # [14]
330
  ]
331
 
332
- # [15 .. 15+D-1] per-domain status lines
333
  for domain, domain_datasets in DOMAIN_GROUPS.items():
334
  outputs.append(build_domain_status_md(domain, domain_datasets, RESULTS_PATH))
335
 
336
  value_tables, rank_tables = _dataset_tables(grouped, datasets)
337
- # [15+D .. 15+D+N-1] value tables
338
  outputs.extend(value_tables)
339
- # [15+D+N .. 15+D+2N-1] rank tables
340
  outputs.extend(rank_tables)
341
  return tuple(outputs)
342
 
@@ -346,9 +296,7 @@ DATASETS: list[str] = grouped_initial["datasets"]
346
  FORECAST_CHOICES = _forecast_choices(RESULTS_PATH, DATASETS)
347
  DOMAIN_GROUPS = group_datasets_by_domain(DATASETS, RESULTS_PATH)
348
  N_DOMAINS = len(DOMAIN_GROUPS)
349
- RANK_HISTORY_MODEL_CHOICES = _rank_history_model_choices()
350
- DEFAULT_RANK_HISTORY_MODEL = _default_rank_history_model()
351
- INITIAL_OUTPUTS = refresh_leaderboard(DEFAULT_RANK_HISTORY_MODEL)
352
 
353
  demo = gr.Blocks(css=custom_css)
354
  with demo:
@@ -364,107 +312,86 @@ with demo:
364
  with gr.Tabs(elem_classes="tab-buttons"):
365
  # ── Overall Tab ──────────────────────────────────────────────────────
366
  with gr.TabItem("Overall"):
367
- live_aggregate_metadata_md = gr.Markdown(
368
- INITIAL_OUTPUTS[12],
369
- elem_classes="markdown-text",
370
- )
371
- gr.Markdown(
372
- "**Live cumulative results (release-history aggregate).** Uses every real release collected up to now. MSE, RMSE, MAPE, CRPS, and Stability are lower-is-better; RTG is higher-is-better; more negative Improvement means errors decrease more consistently over releases.",
373
- elem_classes="markdown-text",
374
- )
375
- live_overall_df = _leaderboard_dataframe(
376
- value=INITIAL_OUTPUTS[13],
377
- label="Up-to-now overall results",
378
- )
379
- gr.Markdown(
380
- "**Live relative ranks (same release history).** Average Rank, Win Rate, and Elo are computed from model-vs-model comparisons on shared live releases, so they summarize relative standing rather than absolute error values.",
381
- elem_classes="markdown-text",
382
- )
383
- live_rank_df = gr.HTML(
384
- value=INITIAL_OUTPUTS[14],
385
- elem_classes="rank-html-output",
386
- )
387
  gr.Markdown(
388
  "**Latest evaluation snapshot**",
389
  elem_classes="markdown-text",
390
  )
391
  gr.Markdown(
392
- "**Evaluated timestamps by domain** · hover for dataset names",
393
  elem_classes="markdown-text",
394
  )
395
- domain_pie_plot = gr.Plot(
396
  value=INITIAL_OUTPUTS[1],
397
- label="Domain distribution",
398
  )
399
  gr.Markdown(
400
- "**Latest snapshot aggregate (not release-history).** Geometric mean of absolute metric values across the current dataset snapshot. MSE/CRPS are lower-is-better; RankScore is higher-is-better. This table is comparable to a static leaderboard snapshot, while the live tables above use temporal release histories.",
401
  elem_classes="markdown-text",
402
  )
403
- overall_values_df = _leaderboard_dataframe(
404
  value=INITIAL_OUTPUTS[2],
405
- label="Metric values and RankScore",
406
  )
407
  gr.Markdown(
408
- "**Latest snapshot ranks.** Per-metric ranks derived from the snapshot values above; lower rank is better.",
409
  elem_classes="markdown-text",
410
  )
411
- overall_ranks_df = gr.HTML(
412
  value=INITIAL_OUTPUTS[3],
413
- elem_classes="rank-html-output",
414
  )
415
 
416
  gr.Markdown(
417
- "**Fixed-window live standings.** Select a model to filter canonical-v1 leaderboards computed over the same trailing wall-clock windows for every model (lower AverageRank is better).",
418
  elem_classes="markdown-text",
419
  )
420
- rank_history_model_dropdown = gr.Dropdown(
421
- choices=RANK_HISTORY_MODEL_CHOICES,
422
- value=DEFAULT_RANK_HISTORY_MODEL,
423
- label="Model",
424
- interactive=True,
425
- )
426
  gr.Markdown(
427
  "**Last 24 hours** · all releases whose target window ended in the trailing 24 hours.",
428
  elem_classes="markdown-text",
429
  )
430
- baseline_history_df = gr.HTML(
 
431
  value=INITIAL_OUTPUTS[4],
432
  elem_classes="rank-html-output",
433
  )
434
- gr.Markdown(
435
- "**Last 7 days** · a fixed trailing seven-day comparison window.",
436
- elem_classes="markdown-text",
437
- )
438
- baseline_weekly_df = gr.HTML(
439
  value=INITIAL_OUTPUTS[5],
440
  elem_classes="rank-html-output",
441
  )
442
  gr.Markdown(
443
- "**Last 30 days** · a fixed trailing thirty-day comparison window.",
444
  elem_classes="markdown-text",
445
  )
446
- baseline_monthly_df = gr.HTML(
 
447
  value=INITIAL_OUTPUTS[6],
448
  elem_classes="rank-html-output",
449
  )
 
 
 
 
 
450
  gr.Markdown(
451
- "**Rank trend** · daily MSE / CRPS / RMSE / MAPE rank history for the selected model.",
452
  elem_classes="markdown-text",
453
  )
454
- rank_trend_plot = gr.Plot(
455
- value=INITIAL_OUTPUTS[7],
456
- label="Rank trend over time",
 
457
  )
458
- rank_history_model_dropdown.change(
459
- fn=_rank_history_outputs,
460
- inputs=[rank_history_model_dropdown],
461
- outputs=[baseline_history_df, baseline_weekly_df, baseline_monthly_df, rank_trend_plot],
462
  )
463
 
464
  # ── GIFT-style grouped result tables ────────────────────────────────
465
  with gr.TabItem("GIFT-style Aggregates"):
466
  aggregate_metadata_md = gr.Markdown(
467
- INITIAL_OUTPUTS[8],
468
  elem_classes="markdown-text",
469
  )
470
  gr.Markdown(
@@ -474,17 +401,17 @@ with demo:
474
  with gr.Tabs():
475
  with gr.TabItem("Prediction Length"):
476
  prediction_length_aggregate_df = _leaderboard_dataframe(
477
- value=INITIAL_OUTPUTS[9],
478
  label="Results on TSFM_Bench aggregated by Prediction Length",
479
  )
480
  with gr.TabItem("Domain"):
481
  domain_aggregate_df = _leaderboard_dataframe(
482
- value=INITIAL_OUTPUTS[10],
483
  label="Results on TSFM_Bench aggregated by Domain",
484
  )
485
  with gr.TabItem("Frequency"):
486
  frequency_aggregate_df = _leaderboard_dataframe(
487
- value=INITIAL_OUTPUTS[11],
488
  label="Results on TSFM_Bench aggregated by Frequency",
489
  )
490
 
@@ -557,27 +484,26 @@ with demo:
557
 
558
  refresh_outputs = [
559
  summary_html,
560
- domain_pie_plot,
561
  overall_values_df,
562
  overall_ranks_df,
563
- baseline_history_df,
564
- baseline_weekly_df,
565
- baseline_monthly_df,
566
- rank_trend_plot,
 
 
 
567
  aggregate_metadata_md,
568
  prediction_length_aggregate_df,
569
  domain_aggregate_df,
570
  frequency_aggregate_df,
571
- live_aggregate_metadata_md,
572
- live_overall_df,
573
- live_rank_df,
574
  *domain_status_mds,
575
  *dataset_value_dfs,
576
  *dataset_rank_dfs,
577
  ]
578
 
579
- demo.load(refresh_leaderboard, inputs=[rank_history_model_dropdown], outputs=refresh_outputs)
580
- gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[rank_history_model_dropdown], outputs=refresh_outputs)
581
 
582
  if __name__ == "__main__":
583
  demo.queue(default_concurrency_limit=20).launch()
 
24
  dataset_section_title,
25
  get_grouped_dfs,
26
  group_datasets_by_domain,
27
+ load_fixed_window_table,
 
 
28
  load_gift_aggregate_metadata_md,
29
  load_gift_aggregate_table,
 
 
30
  load_forecast_snapshots,
31
  make_domain_pie_chart,
32
  make_forecast_plot,
 
33
  prepare_ranks_table,
34
  prepare_values_table,
35
  )
36
 
37
  RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "space/results" if Path("space/results").exists() else "results")
38
  REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
39
+ DOMAIN_OUTPUT_OFFSET = 14
40
  MODEL_COLUMN_WIDTH = "180px"
41
  RANK_MODEL_COLUMN_WIDTH = MODEL_COLUMN_WIDTH
42
  RANK_INDEX_COLUMN_WIDTH = MODEL_COLUMN_WIDTH
43
  METRIC_COLUMN_WIDTH = "118px"
44
  MODEL_COLUMN_NAMES = {"model"}
45
 
46
+ def _fixed_window_outputs(window: str) -> tuple[str, str]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  return (
48
+ _rank_table_html(load_fixed_window_table(RESULTS_PATH, window, "rank")),
49
+ _rank_table_html(load_fixed_window_table(RESULTS_PATH, window, "overall")),
 
 
50
  )
51
 
52
 
 
255
  return update_forecast_plot
256
 
257
 
258
+ def refresh_leaderboard() -> tuple:
259
  grouped = get_grouped_dfs(RESULTS_PATH)
260
  datasets = grouped["datasets"]
261
+ one_day_rank, one_day_metrics = _fixed_window_outputs("1d")
262
+ seven_day_rank, seven_day_metrics = _fixed_window_outputs("7d")
263
+ thirty_day_rank, thirty_day_metrics = _fixed_window_outputs("30d")
264
 
265
  outputs: list = [
266
  build_leaderboard_summary_html(RESULTS_PATH), # [0]
267
+ prepare_values_table(grouped["overall_values"]), # [1]
268
+ _rank_table_html(prepare_ranks_table(grouped["overall_ranks"])), # [2]
269
+ make_domain_pie_chart(RESULTS_PATH), # [3]
270
+ one_day_rank, # [4]
271
+ one_day_metrics, # [5]
272
+ seven_day_rank, # [6]
273
+ seven_day_metrics, # [7]
274
+ thirty_day_rank, # [8]
275
+ thirty_day_metrics, # [9]
276
+ load_gift_aggregate_metadata_md(RESULTS_PATH), # [10]
277
+ load_gift_aggregate_table(RESULTS_PATH, "prediction_length"), # [11]
278
+ load_gift_aggregate_table(RESULTS_PATH, "domain"), # [12]
279
+ load_gift_aggregate_table(RESULTS_PATH, "frequency"), # [13]
 
280
  ]
281
 
282
+ # [14 .. 14+D-1] per-domain status lines
283
  for domain, domain_datasets in DOMAIN_GROUPS.items():
284
  outputs.append(build_domain_status_md(domain, domain_datasets, RESULTS_PATH))
285
 
286
  value_tables, rank_tables = _dataset_tables(grouped, datasets)
287
+ # [14+D .. 14+D+N-1] value tables
288
  outputs.extend(value_tables)
289
+ # [14+D+N .. 14+D+2N-1] rank tables
290
  outputs.extend(rank_tables)
291
  return tuple(outputs)
292
 
 
296
  FORECAST_CHOICES = _forecast_choices(RESULTS_PATH, DATASETS)
297
  DOMAIN_GROUPS = group_datasets_by_domain(DATASETS, RESULTS_PATH)
298
  N_DOMAINS = len(DOMAIN_GROUPS)
299
+ INITIAL_OUTPUTS = refresh_leaderboard()
 
 
300
 
301
  demo = gr.Blocks(css=custom_css)
302
  with demo:
 
312
  with gr.Tabs(elem_classes="tab-buttons"):
313
  # ── Overall Tab ──────────────────────────────────────────────────────
314
  with gr.TabItem("Overall"):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
315
  gr.Markdown(
316
  "**Latest evaluation snapshot**",
317
  elem_classes="markdown-text",
318
  )
319
  gr.Markdown(
320
+ "**Overall summary.** Geometric mean of absolute metric values across the latest dataset snapshot. MSE/CRPS are lower-is-better; RankScore is higher-is-better.",
321
  elem_classes="markdown-text",
322
  )
323
+ overall_values_df = _leaderboard_dataframe(
324
  value=INITIAL_OUTPUTS[1],
325
+ label="Metric values and RankScore",
326
  )
327
  gr.Markdown(
328
+ "**Latest snapshot ranks.** Per-metric ranks derived from the snapshot values above; lower rank is better.",
329
  elem_classes="markdown-text",
330
  )
331
+ overall_ranks_df = gr.HTML(
332
  value=INITIAL_OUTPUTS[2],
333
+ elem_classes="rank-html-output",
334
  )
335
  gr.Markdown(
336
+ "**Evaluated timestamps by domain** · hover for dataset names",
337
  elem_classes="markdown-text",
338
  )
339
+ domain_pie_plot = gr.Plot(
340
  value=INITIAL_OUTPUTS[3],
341
+ label="Domain distribution",
342
  )
343
 
344
  gr.Markdown(
345
+ "**Pairwise historical ranking.** MSE and CRPS wins are computed only on shared releases, balanced equally across datasets. Official ranks require at least 30 shared releases, 5 shared datasets, 7 days of shared coverage, 3 eligible opponents, and membership in the main comparison component. Absolute metrics remain descriptive and do not determine rank.",
346
  elem_classes="markdown-text",
347
  )
 
 
 
 
 
 
348
  gr.Markdown(
349
  "**Last 24 hours** · all releases whose target window ended in the trailing 24 hours.",
350
  elem_classes="markdown-text",
351
  )
352
+ gr.Markdown("**Ranking**", elem_classes="markdown-text")
353
+ last_24h_rank_df = gr.HTML(
354
  value=INITIAL_OUTPUTS[4],
355
  elem_classes="rank-html-output",
356
  )
357
+ gr.Markdown("**Metrics**", elem_classes="markdown-text")
358
+ last_24h_metrics_df = gr.HTML(
 
 
 
359
  value=INITIAL_OUTPUTS[5],
360
  elem_classes="rank-html-output",
361
  )
362
  gr.Markdown(
363
+ "**Last 7 days** · a fixed trailing seven-day comparison window.",
364
  elem_classes="markdown-text",
365
  )
366
+ gr.Markdown("**Ranking**", elem_classes="markdown-text")
367
+ last_7d_rank_df = gr.HTML(
368
  value=INITIAL_OUTPUTS[6],
369
  elem_classes="rank-html-output",
370
  )
371
+ gr.Markdown("**Metrics**", elem_classes="markdown-text")
372
+ last_7d_metrics_df = gr.HTML(
373
+ value=INITIAL_OUTPUTS[7],
374
+ elem_classes="rank-html-output",
375
+ )
376
  gr.Markdown(
377
+ "**Last 30 days** · a fixed trailing thirty-day comparison window.",
378
  elem_classes="markdown-text",
379
  )
380
+ gr.Markdown("**Ranking**", elem_classes="markdown-text")
381
+ last_30d_rank_df = gr.HTML(
382
+ value=INITIAL_OUTPUTS[8],
383
+ elem_classes="rank-html-output",
384
  )
385
+ gr.Markdown("**Metrics**", elem_classes="markdown-text")
386
+ last_30d_metrics_df = gr.HTML(
387
+ value=INITIAL_OUTPUTS[9],
388
+ elem_classes="rank-html-output",
389
  )
390
 
391
  # ── GIFT-style grouped result tables ────────────────────────────────
392
  with gr.TabItem("GIFT-style Aggregates"):
393
  aggregate_metadata_md = gr.Markdown(
394
+ INITIAL_OUTPUTS[10],
395
  elem_classes="markdown-text",
396
  )
397
  gr.Markdown(
 
401
  with gr.Tabs():
402
  with gr.TabItem("Prediction Length"):
403
  prediction_length_aggregate_df = _leaderboard_dataframe(
404
+ value=INITIAL_OUTPUTS[11],
405
  label="Results on TSFM_Bench aggregated by Prediction Length",
406
  )
407
  with gr.TabItem("Domain"):
408
  domain_aggregate_df = _leaderboard_dataframe(
409
+ value=INITIAL_OUTPUTS[12],
410
  label="Results on TSFM_Bench aggregated by Domain",
411
  )
412
  with gr.TabItem("Frequency"):
413
  frequency_aggregate_df = _leaderboard_dataframe(
414
+ value=INITIAL_OUTPUTS[13],
415
  label="Results on TSFM_Bench aggregated by Frequency",
416
  )
417
 
 
484
 
485
  refresh_outputs = [
486
  summary_html,
 
487
  overall_values_df,
488
  overall_ranks_df,
489
+ domain_pie_plot,
490
+ last_24h_rank_df,
491
+ last_24h_metrics_df,
492
+ last_7d_rank_df,
493
+ last_7d_metrics_df,
494
+ last_30d_rank_df,
495
+ last_30d_metrics_df,
496
  aggregate_metadata_md,
497
  prediction_length_aggregate_df,
498
  domain_aggregate_df,
499
  frequency_aggregate_df,
 
 
 
500
  *domain_status_mds,
501
  *dataset_value_dfs,
502
  *dataset_rank_dfs,
503
  ]
504
 
505
+ demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
506
+ gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
507
 
508
  if __name__ == "__main__":
509
  demo.queue(default_concurrency_limit=20).launch()