Spaces:
Runtime error
Runtime error
Download app.py from ThinkcatLab/LiveHouse-TS: direct link, hf CLI and curl.
- Browser
- Download file 15.1 kB
-
https://huggingface.co/spaces/ThinkcatLab/LiveHouse-TS/resolve/977458d0a8bd2788bef8a430e1b72bb4e3d77cf0/app.py
- Command line
-
hf download hf://spaces/ThinkcatLab/LiveHouse-TS@977458d0a8bd2788bef8a430e1b72bb4e3d77cf0/app.py
-
curl -L -o app.py https://huggingface.co/spaces/ThinkcatLab/LiveHouse-TS/resolve/977458d0a8bd2788bef8a430e1b72bb4e3d77cf0/app.py
15.1 kB
| import json | |
| import os | |
| import re | |
| from pathlib import Path | |
| import gradio as gr | |
| from src.about import ( | |
| CITATION_BUTTON_LABEL, | |
| CITATION_BUTTON_TEXT, | |
| INTRODUCTION_TEXT, | |
| LLM_BENCHMARKS_TEXT, | |
| TITLE, | |
| ) | |
| from src.display.css_html_js import custom_css | |
| from src.utils import ( | |
| build_leaderboard_summary_html, | |
| build_domain_status_md, | |
| dataset_section_title, | |
| get_grouped_dfs, | |
| group_datasets_by_domain, | |
| load_baseline_rank_history, | |
| load_baseline_rank_history_weekly, | |
| load_forecast_snapshots, | |
| load_gift_aggregate_metadata_md, | |
| load_gift_aggregate_table, | |
| load_live_aggregate_metadata_md, | |
| load_live_aggregate_table, | |
| make_domain_pie_chart, | |
| make_forecast_plot, | |
| make_rank_trend_plot, | |
| prepare_ranks_table, | |
| prepare_values_table, | |
| ) | |
| RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "space/results" if Path("space/results").exists() else "results") | |
| REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300")) | |
| DOMAIN_OUTPUT_OFFSET = 13 | |
| def _dataset_tables(grouped: dict, datasets: list[str]) -> tuple[list, list]: | |
| values_tables = [ | |
| prepare_values_table(grouped["dataset_values"].get(dataset, None)) | |
| for dataset in datasets | |
| ] | |
| rank_tables = [ | |
| prepare_ranks_table(grouped["dataset_ranks"].get(dataset, None)) | |
| for dataset in datasets | |
| ] | |
| return values_tables, rank_tables | |
| def _get_model_slugs(results_path: str) -> list[str]: | |
| """Return subdirs of results_path that contain a forecasts/ subdirectory.""" | |
| slugs = [] | |
| for p in sorted(Path(results_path).iterdir()): | |
| if p.is_dir() and (p / "forecasts").exists(): | |
| slugs.append(p.name) | |
| return slugs | |
| def _build_forecast_choices(results_path: str) -> dict[str, dict[str, dict]]: | |
| """Build {model_slug: {dataset_name: snapshot}} for all models.""" | |
| choices: dict[str, dict[str, dict]] = {} | |
| for slug in _get_model_slugs(results_path): | |
| snaps = load_forecast_snapshots(results_path, slug) | |
| if snaps: | |
| choices[slug] = snaps | |
| return choices | |
| def _short_dataset_key(dataset_name: str) -> str: | |
| """Extract the base dataset key from a leaderboard dataset name like 'ds/freq/term'.""" | |
| return dataset_name.split("/")[0] | |
| def refresh_leaderboard() -> tuple: | |
| grouped = get_grouped_dfs(RESULTS_PATH) | |
| datasets = grouped["datasets"] | |
| daily_df = load_baseline_rank_history(RESULTS_PATH) | |
| outputs: list = [ | |
| make_domain_pie_chart(RESULTS_PATH), # [0] domain pie chart | |
| prepare_values_table(grouped["overall_values"]), # [1] | |
| prepare_ranks_table(grouped["overall_ranks"]), # [2] | |
| daily_df, # [3] daily table | |
| load_baseline_rank_history_weekly(RESULTS_PATH), # [4] weekly table | |
| make_rank_trend_plot(daily_df), # [5] rank trend plot | |
| load_gift_aggregate_metadata_md(RESULTS_PATH), # [6] | |
| load_gift_aggregate_table(RESULTS_PATH, "prediction_length"), # [7] | |
| load_gift_aggregate_table(RESULTS_PATH, "domain"), # [8] | |
| load_gift_aggregate_table(RESULTS_PATH, "frequency"), # [9] | |
| load_live_aggregate_metadata_md(RESULTS_PATH), # [10] | |
| load_live_aggregate_table(RESULTS_PATH, "overall"), # [11] | |
| load_live_aggregate_table(RESULTS_PATH, "rank"), # [12] | |
| ] | |
| # [10 .. 10+D-1] per-domain status lines | |
| for domain, domain_datasets in DOMAIN_GROUPS.items(): | |
| outputs.append(build_domain_status_md(domain, domain_datasets, RESULTS_PATH)) | |
| value_tables, rank_tables = _dataset_tables(grouped, datasets) | |
| # [10+D .. 10+D+N-1] value tables | |
| outputs.extend(value_tables) | |
| # [10+D+N .. 10+D+2N-1] rank tables | |
| outputs.extend(rank_tables) | |
| return tuple(outputs) | |
| grouped_initial = get_grouped_dfs(RESULTS_PATH) | |
| DATASETS: list[str] = grouped_initial["datasets"] | |
| DOMAIN_GROUPS = group_datasets_by_domain(DATASETS, RESULTS_PATH) | |
| FORECAST_CHOICES = _build_forecast_choices(RESULTS_PATH) | |
| MODEL_SLUGS = list(FORECAST_CHOICES.keys()) | |
| N_DOMAINS = len(DOMAIN_GROUPS) | |
| INITIAL_OUTPUTS = refresh_leaderboard() | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| gr.HTML(TITLE) | |
| gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
| gr.HTML(build_leaderboard_summary_html(RESULTS_PATH)) | |
| dataset_value_dfs: list[gr.Dataframe] = [] | |
| dataset_rank_dfs: list[gr.Dataframe] = [] | |
| domain_status_mds: list[gr.Markdown] = [] | |
| dataset_idx = 0 | |
| with gr.Tabs(elem_classes="tab-buttons"): | |
| # ββ Overall Tab ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| with gr.TabItem("Overall"): | |
| live_aggregate_metadata_md = gr.Markdown( | |
| INITIAL_OUTPUTS[10], | |
| elem_classes="markdown-text", | |
| ) | |
| gr.Markdown( | |
| "**Cumulative live evaluation.** MSE, RMSE, MAPE, CRPS, and Stability are lower-is-better; RTG is higher-is-better; more negative Improvement indicates a stronger decreasing-error trend.", | |
| elem_classes="markdown-text", | |
| ) | |
| live_overall_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[11], | |
| interactive=False, | |
| wrap=True, | |
| label="Up-to-now overall results", | |
| ) | |
| live_rank_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[12], | |
| interactive=False, | |
| wrap=True, | |
| label="Average Rank, Win Rate, and Elo", | |
| ) | |
| gr.Markdown( | |
| "**Latest evaluation snapshot**", | |
| elem_classes="markdown-text", | |
| ) | |
| gr.Markdown( | |
| "**Evaluated timestamps by domain** Β· hover for dataset names", | |
| elem_classes="markdown-text", | |
| ) | |
| domain_pie_plot = gr.Plot( | |
| value=INITIAL_OUTPUTS[0], | |
| label="Domain distribution", | |
| ) | |
| gr.Markdown( | |
| "Geometric mean of **absolute** metrics across all datasets. MSE/CRPS are lower-is-better; RankScore is higher-is-better.", | |
| elem_classes="markdown-text", | |
| ) | |
| overall_values_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[1], | |
| interactive=False, | |
| wrap=True, | |
| label="Metric values and RankScore", | |
| ) | |
| overall_ranks_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[2], | |
| interactive=False, | |
| wrap=True, | |
| label="Ranks (average per-dataset rank, lower is better)", | |
| ) | |
| gr.Markdown( | |
| "**Baseline daily rank history** Β· average rank across all datasets per day (lower is better)", | |
| elem_classes="markdown-text", | |
| ) | |
| baseline_history_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[3], | |
| interactive=False, | |
| wrap=True, | |
| label="Baseline rank history β by day", | |
| ) | |
| gr.Markdown( | |
| "**Baseline weekly rank history** Β· average of daily ranks per ISO week (lower is better)", | |
| elem_classes="markdown-text", | |
| ) | |
| baseline_weekly_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[4], | |
| interactive=False, | |
| wrap=True, | |
| label="Baseline rank history β by week", | |
| ) | |
| gr.Markdown( | |
| "**Baseline rank trend** Β· MSE / CRPS / MASE / MAE rank over time", | |
| elem_classes="markdown-text", | |
| ) | |
| rank_trend_plot = gr.Plot( | |
| value=INITIAL_OUTPUTS[5], | |
| label="Rank trend over time", | |
| ) | |
| # ββ GIFT-style grouped result tables ββββββββββββββββββββββββββββββββ | |
| with gr.TabItem("GIFT-style Aggregates"): | |
| aggregate_metadata_md = gr.Markdown( | |
| INITIAL_OUTPUTS[6], | |
| elem_classes="markdown-text", | |
| ) | |
| gr.Markdown( | |
| "MSE and CRPS are normalized per dataset configuration against Seasonal-Naive (1.0 = baseline). Rank is the mean per-configuration CRPS rank. Lower is better for all three metrics.", | |
| elem_classes="markdown-text", | |
| ) | |
| with gr.Tabs(): | |
| with gr.TabItem("Prediction Length"): | |
| prediction_length_aggregate_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[7], | |
| interactive=False, | |
| wrap=True, | |
| label="Results on TSFM_Bench aggregated by Prediction Length", | |
| ) | |
| with gr.TabItem("Domain"): | |
| domain_aggregate_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[8], | |
| interactive=False, | |
| wrap=True, | |
| label="Results on TSFM_Bench aggregated by Domain", | |
| ) | |
| with gr.TabItem("Frequency"): | |
| frequency_aggregate_df = gr.Dataframe( | |
| value=INITIAL_OUTPUTS[9], | |
| interactive=False, | |
| wrap=True, | |
| label="Results on TSFM_Bench aggregated by Frequency", | |
| ) | |
| # ββ Domain Tabs βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| domain_idx = 0 | |
| for domain, domain_datasets in DOMAIN_GROUPS.items(): | |
| with gr.TabItem(domain): | |
| # Per-domain status line (data source, last eval, refresh intervals) | |
| domain_status_mds.append( | |
| gr.Markdown( | |
| INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + domain_idx], | |
| elem_classes="markdown-text", | |
| ) | |
| ) | |
| domain_idx += 1 | |
| for dataset in domain_datasets: | |
| gr.Markdown( | |
| dataset_section_title(dataset, RESULTS_PATH), | |
| elem_classes="dataset-heading", | |
| ) | |
| dataset_value_dfs.append( | |
| gr.Dataframe( | |
| value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + dataset_idx], | |
| interactive=False, | |
| wrap=True, | |
| show_label=False, | |
| ) | |
| ) | |
| gr.Markdown( | |
| dataset_section_title(dataset, RESULTS_PATH, ranks=True), | |
| elem_classes="dataset-heading", | |
| ) | |
| dataset_rank_dfs.append( | |
| gr.Dataframe( | |
| value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + len(DATASETS) + dataset_idx], | |
| interactive=False, | |
| wrap=True, | |
| show_label=False, | |
| ) | |
| ) | |
| # ββ Forecast plot for this dataset ββββββββββββββββββββββ | |
| if MODEL_SLUGS: | |
| ds_key = _short_dataset_key(dataset) | |
| # Find which models have a snapshot for this dataset | |
| available_models = [ | |
| slug for slug in MODEL_SLUGS | |
| if any( | |
| _short_dataset_key(k) == ds_key | |
| for k in FORECAST_CHOICES.get(slug, {}) | |
| ) | |
| ] | |
| if available_models: | |
| gr.Markdown( | |
| "**Forecast: true vs predicted** Β· select model", | |
| elem_classes="dataset-heading", | |
| ) | |
| model_dd = gr.Dropdown( | |
| choices=available_models, | |
| value=available_models[0], | |
| label="Model", | |
| interactive=True, | |
| ) | |
| def _get_initial_snapshot(slug, dk=ds_key): | |
| snaps = FORECAST_CHOICES.get(slug, {}) | |
| for name, snap in snaps.items(): | |
| if _short_dataset_key(name) == dk: | |
| return snap | |
| return None | |
| forecast_plot = gr.Plot( | |
| value=make_forecast_plot(_get_initial_snapshot(available_models[0])), | |
| label="True vs Predicted", | |
| ) | |
| def _make_update_fn(dk): | |
| def _update(slug): | |
| snaps = FORECAST_CHOICES.get(slug, {}) | |
| for name, snap in snaps.items(): | |
| if _short_dataset_key(name) == dk: | |
| return make_forecast_plot(snap) | |
| return None | |
| return _update | |
| model_dd.change( | |
| fn=_make_update_fn(ds_key), | |
| inputs=[model_dd], | |
| outputs=[forecast_plot], | |
| ) | |
| dataset_idx += 1 | |
| with gr.Accordion("About", open=False): | |
| gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") | |
| with gr.Accordion("Citation", open=False): | |
| gr.Textbox( | |
| value=CITATION_BUTTON_TEXT, | |
| label=CITATION_BUTTON_LABEL, | |
| lines=12, | |
| show_copy_button=True, | |
| ) | |
| refresh_outputs = [ | |
| domain_pie_plot, | |
| overall_values_df, | |
| overall_ranks_df, | |
| baseline_history_df, | |
| baseline_weekly_df, | |
| rank_trend_plot, | |
| aggregate_metadata_md, | |
| prediction_length_aggregate_df, | |
| domain_aggregate_df, | |
| frequency_aggregate_df, | |
| live_aggregate_metadata_md, | |
| live_overall_df, | |
| live_rank_df, | |
| *domain_status_mds, | |
| *dataset_value_dfs, | |
| *dataset_rank_dfs, | |
| ] | |
| demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs) | |
| gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[], outputs=refresh_outputs) | |
| if __name__ == "__main__": | |
| demo.queue(default_concurrency_limit=20).launch() | |