Update online Open-Meteo eval results

app.py

@@ -1,3 +1,7 @@
+import json
+import os
+from pathlib import Path
+
 import gradio as gr
 
 from src.about import (
@@ -11,9 +15,16 @@ from src.display.css_html_js import custom_css
 from src.populate import get_model_info_df
 from src.utils import get_grouped_dfs, get_merged_df, prepare_leaderboard_table
 
-RESULTS_PATH = "results"
-grouped = get_grouped_dfs(RESULTS_PATH)
-model_info_df = get_model_info_df(RESULTS_PATH)
+RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
+REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
+AUTO_RUN_EVAL = os.getenv("TSFM_AUTO_RUN_EVAL", "0") == "1"
+
+
+def load_online_status(results_path: str = RESULTS_PATH) -> dict:
+    status_path = Path(results_path) / "online_status.json"
+    if not status_path.exists():
+        return {"status": "unknown", "message": "No online_status.json yet. Run run_online_eval.py."}
+    return json.loads(status_path.read_text())
 
 
 def build_table(result_df, info_df, sort_col=None, metric_only=False):
@@ -41,38 +52,71 @@ def build_table(result_df, info_df, sort_col=None, metric_only=False):
     return merged[cols].reset_index(drop=True)
 
 
-overall_table = build_table(
-    prepare_leaderboard_table(grouped["overall"]), model_info_df, sort_col="MASE_Rank"
-)
-domain_table = build_table(
-    prepare_leaderboard_table(grouped["domain"], "domain"),
-    model_info_df,
-    sort_col=None,
-    metric_only=True,
-)
-frequency_table = build_table(
-    prepare_leaderboard_table(grouped["frequency"], "frequency"),
-    model_info_df,
-    sort_col=None,
-    metric_only=True,
-)
+def refresh_leaderboard():
+    grouped = get_grouped_dfs(RESULTS_PATH)
+    model_info_df = get_model_info_df(RESULTS_PATH)
+    status = load_online_status(RESULTS_PATH)
+
+    overall = build_table(
+        prepare_leaderboard_table(grouped["overall"]),
+        model_info_df,
+        sort_col="MASE_Rank",
+    )
+    domain = build_table(
+        prepare_leaderboard_table(grouped["domain"], "domain"),
+        model_info_df,
+        sort_col=None,
+        metric_only=True,
+    )
+    frequency = build_table(
+        prepare_leaderboard_table(grouped["frequency"], "frequency"),
+        model_info_df,
+        sort_col=None,
+        metric_only=True,
+    )
+
+    finished = status.get("finished_at", "n/a")
+    data_source = status.get("data_source", "n/a")
+    status_line = (
+        f"**Live benchmark** · Data: `{data_source}` (Open-Meteo) · "
+        f"Last eval: `{finished}` · Auto-refresh every {REFRESH_SECONDS}s"
+    )
+    return overall, domain, frequency, status_line
+
+
+def trigger_online_eval():
+    return (
+        gr.update(),
+        gr.update(),
+        gr.update(),
+        "**Run eval locally:** `python scripts/run_online_eval.py` then push `space/results/` to HF Space. "
+        "Set `TSFM_API_KEY` in your environment.",
+    )
+
+
+overall_table, domain_table, frequency_table, status_md = refresh_leaderboard()
 
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    status = gr.Markdown(status_md, elem_classes="markdown-text")
+
+    with gr.Row():
+        refresh_btn = gr.Button("Refresh now", variant="secondary")
+        eval_btn = gr.Button("Run online eval", variant="primary")
 
     with gr.Tabs(elem_classes="tab-buttons"):
         with gr.TabItem("Overall"):
-            gr.Dataframe(
+            overall_df = gr.Dataframe(
                 value=overall_table,
                 interactive=False,
                 wrap=True,
-                label="Overall Leaderboard (normalized vs TSFM2, lower is better)",
+                label="Overall (normalized vs TSFM2, lower is better)",
             )
 
         with gr.TabItem("By Domain"):
-            gr.Dataframe(
+            domain_df = gr.Dataframe(
                 value=domain_table,
                 interactive=False,
                 wrap=True,
@@ -80,7 +124,7 @@ with demo:
             )
 
         with gr.TabItem("By Frequency"):
-            gr.Dataframe(
+            frequency_df = gr.Dataframe(
                 value=frequency_table,
                 interactive=False,
                 wrap=True,
@@ -99,4 +143,18 @@ with demo:
                 show_copy_button=True,
             )
 
+    refresh_btn.click(
+        refresh_leaderboard,
+        outputs=[overall_df, domain_df, frequency_df, status],
+    )
+    eval_btn.click(
+        trigger_online_eval,
+        outputs=[overall_df, domain_df, frequency_df, status],
+    )
+    demo.load(refresh_leaderboard, outputs=[overall_df, domain_df, frequency_df, status])
+    gr.Timer(value=REFRESH_SECONDS).tick(
+        refresh_leaderboard,
+        outputs=[overall_df, domain_df, frequency_df, status],
+    )
+
 demo.queue(default_concurrency_limit=20).launch()

results/dataset_properties.csv

@@ -1,7 +1,4 @@
-dataset,domain,frequency,num_variates
-synthetic_energy,Energy,H,4
-synthetic_finance,Econ/Fin,D,2
-synthetic_healthcare,Healthcare,D,1
-synthetic_retail,Sales,W,3
-synthetic_transport,Transport,15T,5
-synthetic_cloudops,Web/CloudOps,H,2
+dataset,domain,frequency,num_variates
+berlin,Energy,H,3
+nyc,Econ/Fin,H,3
+tokyo,Transport,H,3

results/online_status.json

@@ -0,0 +1,85 @@
+{
+    "status": "ok",
+    "started_at": "2026-05-28T09:33:12.561302+00:00",
+    "finished_at": "2026-05-28T09:35:40.815492+00:00",
+    "data_source": "open_meteo",
+    "data_config": "configs/datasets/open_meteo.yaml",
+    "baseline_model": "TSFM2",
+    "models": {
+        "TSFM1": {
+            "model": "TSFM1",
+            "api_model_id": "amazon/chronos-bolt-tiny",
+            "evaluated_at": "2026-05-28T09:34:09.774442+00:00",
+            "datasets": [
+                {
+                    "dataset": "berlin/H",
+                    "data_fetched_at": "2026-05-28T09:33:20.812480+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                },
+                {
+                    "dataset": "nyc/H",
+                    "data_fetched_at": "2026-05-28T09:33:56.802373+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                },
+                {
+                    "dataset": "tokyo/H",
+                    "data_fetched_at": "2026-05-28T09:34:04.318782+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                }
+            ]
+        },
+        "TSFM2": {
+            "model": "TSFM2",
+            "api_model_id": "amazon/chronos-bolt-base",
+            "evaluated_at": "2026-05-28T09:35:12.902197+00:00",
+            "datasets": [
+                {
+                    "dataset": "berlin/H",
+                    "data_fetched_at": "2026-05-28T09:34:12.472680+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                },
+                {
+                    "dataset": "nyc/H",
+                    "data_fetched_at": "2026-05-28T09:34:24.600161+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                },
+                {
+                    "dataset": "tokyo/H",
+                    "data_fetched_at": "2026-05-28T09:35:08.709864+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                }
+            ]
+        },
+        "TSFM3": {
+            "model": "TSFM3",
+            "api_model_id": "amazon/chronos-2",
+            "evaluated_at": "2026-05-28T09:35:40.814222+00:00",
+            "datasets": [
+                {
+                    "dataset": "berlin/H",
+                    "data_fetched_at": "2026-05-28T09:35:16.938802+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                },
+                {
+                    "dataset": "nyc/H",
+                    "data_fetched_at": "2026-05-28T09:35:27.945067+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                },
+                {
+                    "dataset": "tokyo/H",
+                    "data_fetched_at": "2026-05-28T09:35:36.696839+00:00",
+                    "context_length": 504,
+                    "prediction_length": 24
+                }
+            ]
+        }
+    }
+}

results/tsfm1/all_results.csv

@@ -1,13 +1,4 @@
 dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
-synthetic_cloudops/H/short,TSFM1,0.5971685558130638,0.5971685558130638,0.6258273616249179,1.2305884493234924,0.00825023486671363,0.008302139742829246,7.529956407038758,0.712455803851091,0.00939916018625336,0.008256303884475546,0.006157524551587105,Web/CloudOps,2
-synthetic_cloudops/H/medium,TSFM1,28.219417888862992,28.219417888862992,4.2863336140977015,8.427977828154779,0.055830729955831555,0.058290338598782694,215.64986172414854,4.897601985210062,0.06502390191024007,0.05690828640614811,0.05195070874205873,Web/CloudOps,2
-synthetic_energy/H/short,TSFM1,0.5938844795508932,0.5938844795508932,0.6232670075468945,1.225933804288643,0.006576108972318304,0.006609134861303338,9.430287036899557,0.7104940588198181,0.007500291420809269,0.006579483855702363,0.00531288081040946,Energy,4
-synthetic_energy/H/medium,TSFM1,28.177925692725847,28.177925692725847,4.28214992023011,8.421720524997843,0.046736440531403224,0.0484581455670345,193.34833927952195,4.894000085698505,0.054131364332840645,0.04736383600343495,0.0424471675512218,Energy,4
-synthetic_energy/H/long,TSFM1,61.80216253092821,61.80216253092821,6.323331954638164,12.436058586785576,0.06655727200389391,0.07009315069403968,347.4559498304473,7.247885081269499,0.07809448854763935,0.06813261653258963,0.06319847428791797,Energy,4
-synthetic_finance/D/short,TSFM1,0.21462321215682875,0.21462321215682875,0.34737526961735316,1.2292518849597884,0.015079765180756143,0.015236813989533212,8.673370801589083,0.42711793492348743,0.018655986639533738,0.015172924990955227,0.011858844878516494,Econ/Fin,2
-synthetic_healthcare/D/short,TSFM1,1.0257587935734411,1.0257587935734411,0.7639493531121148,2.047133523345588,0.02006523328012893,0.0202472091868943,21.18515881410396,0.9337531657442585,0.024553777303309817,0.020088651878755682,0.016232297383302122,Healthcare,1
-synthetic_healthcare/D/medium,TSFM1,13.161946064299542,13.161946064299542,2.918498039891985,7.82600162057896,0.07665831003903978,0.0813606331874339,230.98259241986682,3.344795084105245,0.09006312772966453,0.0785845037248018,0.07368690771688535,Healthcare,1
-synthetic_retail/W/short,TSFM1,0.15335441406544495,0.15335441406544495,0.2897797259966532,1.0250723027020774,0.03787222523447689,0.03818369038110213,20.567296838161607,0.3610418977842159,0.046965955166542304,0.03769585109888242,0.03303350288597904,Sales,3
-synthetic_transport/15T/short,TSFM1,3.24074837797425,3.24074837797425,1.6318799565816537,0.8499323196969463,0.01436678256288863,0.014493578040926292,2.9683604599837,1.659709649691208,0.014601116568078498,0.014356287845643792,0.009468138017080897,Transport,5
-synthetic_transport/15T/medium,TSFM1,34.57169887739867,34.57169887739867,4.895799042936166,2.549874253372172,0.04397258062265444,0.04539918535947395,53.55165953407358,5.4208803755283945,0.04914222674096303,0.04438217595286957,0.039107729585810255,Transport,5
-synthetic_transport/15T/long,TSFM1,71.9796988118075,71.9796988118075,6.964037973954942,3.6271054430859984,0.0607152509221608,0.06353083103560946,95.21107453470914,7.821939912198021,0.06939865639724888,0.0617870865689229,0.05659354995119928,Transport,5
+berlin/H/short,TSFM1,26.466026654661675,26.466026654661675,3.9281906419330173,0.801173140704702,0.2664836929777801,0.2181852629665049,4.3784708273154,5.144514229221421,0.20596365915123496,0.15726742747849448,0.11943187145790482,Energy,1
+nyc/H/short,TSFM1,61.155471274035534,61.155471274035534,5.074739774068196,0.5313184802655554,0.13086885729110534,0.11411951426415046,3.6021412314581593,7.820196370554613,0.021813912129850066,0.014155645493788019,0.011044288087260955,Econ/Fin,1
+tokyo/H/short,TSFM1,4.9888988149732105,4.9888988149732105,1.3425839688732393,0.5246587213755459,0.3603745541647132,0.8183498665142726,4.24507431889489,2.233584297709225,0.24657786181385902,0.14821535264635233,0.1081772060090852,Transport,1

results/tsfm1/config.json

@@ -2,9 +2,10 @@
     "model": "TSFM1",
     "model_type": "zero-shot",
     "model_dtype": "float32",
-    "model_link": "https://github.com/zhouziyu02/TSFM_Bench",
-    "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/generate_demo_leaderboard.py",
-    "org": "Demo",
+    "model_link": "https://tsfm.ai/",
+    "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
+    "org": "TSFM.ai",
     "testdata_leakage": "No",
-    "replication_code_available": "Yes"
+    "replication_code_available": "Yes",
+    "api_model_id": "amazon/chronos-bolt-tiny"
 }

results/tsfm1/online_meta.json

@@ -0,0 +1,25 @@
+{
+    "model": "TSFM1",
+    "api_model_id": "amazon/chronos-bolt-tiny",
+    "evaluated_at": "2026-05-28T09:34:09.774442+00:00",
+    "datasets": [
+        {
+            "dataset": "berlin/H",
+            "data_fetched_at": "2026-05-28T09:33:20.812480+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        },
+        {
+            "dataset": "nyc/H",
+            "data_fetched_at": "2026-05-28T09:33:56.802373+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        },
+        {
+            "dataset": "tokyo/H",
+            "data_fetched_at": "2026-05-28T09:34:04.318782+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        }
+    ]
+}

results/tsfm2/all_results.csv

@@ -1,13 +1,4 @@
 dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
-synthetic_cloudops/H/short,TSFM2,0.7025512421330163,0.7025512421330163,0.7362674842646093,1.4477511168511676,0.009706158666721918,0.009767223226857937,8.85877224357501,0.8381832986483424,0.011057835513239248,0.00971329868761829,0.007244146531278948,Web/CloudOps,2
-synthetic_cloudops/H/medium,TSFM2,33.199315163368226,33.199315163368226,5.042745428350237,9.915268033123269,0.06568321171274301,0.06857686893974435,253.70571967546886,5.761884688482426,0.07649870812969421,0.06695092518370366,0.06111848087301028,Web/CloudOps,2
-synthetic_energy/H/short,TSFM2,0.6986876230010508,0.6986876230010508,0.7332553029963464,1.4422750638689918,0.00773659879096271,0.007775452778003927,11.094455337528892,0.8358753633174331,0.00882387225977561,0.00774056924200278,0.006250448012246424,Energy,4
-synthetic_energy/H/medium,TSFM2,33.150500814971586,33.150500814971586,5.037823435564836,9.907906499997463,0.05498404768400379,0.05700958302004058,227.46863444649642,5.7576471596453,0.06368395803863605,0.05572216000404112,0.049937844177907996,Energy,4
-synthetic_energy/H/long,TSFM2,72.70842650697436,72.70842650697436,7.4392140642801925,14.630657160924207,0.07830267294575755,0.08246253022828198,408.7717056828792,8.52692362502294,0.09187586887957572,0.08015601945010545,0.07435114622107997,Energy,4
-synthetic_finance/D/short,TSFM2,0.25249789665509265,0.25249789665509265,0.4086767877851214,1.4461786881879863,0.017740900212654287,0.017925663517097897,10.203965648928333,0.5024916881452793,0.021948219575922044,0.01785049998935909,0.013951582210019405,Econ/Fin,2
-synthetic_healthcare/D/short,TSFM2,1.2067750512628719,1.2067750512628719,0.8987639448377821,2.408392380406574,0.02360615680015168,0.02382024610222859,24.923716251887015,1.098533136169716,0.028886796827423317,0.023633708092653742,0.019096820450943675,Healthcare,1
-synthetic_healthcare/D/medium,TSFM2,15.484642428587698,15.484642428587698,3.4335271057552763,9.207060730092895,0.09018624710475268,0.09571839198521635,271.7442263763139,3.9350530401238175,0.10595662085842886,0.09245235732329624,0.08669047966692395,Healthcare,1
-synthetic_retail/W/short,TSFM2,0.1804169577240529,0.1804169577240529,0.34091732470194497,1.2059674149436206,0.04455555909938458,0.044921988683649564,24.19681980960189,0.4247551738637834,0.055254064901814476,0.04434806011633226,0.03886294457174005,Sales,3
-synthetic_transport/15T/short,TSFM2,3.8126451505579415,3.8126451505579415,1.9198587724490044,0.9999203761140545,0.016902097132810153,0.017051268283442698,3.492188776451412,1.9525995878720095,0.01717778419773941,0.016889750406639755,0.011138985902448113,Transport,5
-synthetic_transport/15T/medium,TSFM2,40.67258691458667,40.67258691458667,5.759763579924901,2.999852062790791,0.05173244779135817,0.05341080630526347,63.00195239302774,6.377506324151052,0.05781438440113298,0.05221432465043479,0.046009093630365004,Transport,5
-synthetic_transport/15T/long,TSFM2,84.68199860212647,84.68199860212647,8.192985851711697,4.267182874218822,0.071429706967248,0.07474215415954054,112.0130288643637,9.20228224964473,0.08164547811441045,0.07269069008108577,0.06658064700141092,Transport,5
+berlin/H/short,TSFM2,18.228861802930954,18.228861802930954,2.9531471265686884,0.5820599244735587,0.14223163408323697,0.13481896730470858,3.4027020979674396,4.269527117015532,0.17093303442938024,0.11823098568668831,0.10213142932311715,Energy,1
+nyc/H/short,TSFM2,152.93628831881657,152.93628831881657,8.336753845214844,0.8240694730252156,0.22302551307881188,0.18185803856345417,6.738071750565663,12.366741216618733,0.03449619337535691,0.023254814484256852,0.015018103802445107,Econ/Fin,1
+tokyo/H/short,TSFM2,8.307242700559431,8.307242700559431,1.8618432140081294,0.6809096260138134,0.5041955416902912,0.8758267896424199,5.4925886624835085,2.88222877311282,0.318185352960008,0.20553928464379514,0.13544487699281893,Transport,1

results/tsfm2/config.json

@@ -2,9 +2,10 @@
     "model": "TSFM2",
     "model_type": "zero-shot",
     "model_dtype": "float32",
-    "model_link": "https://github.com/zhouziyu02/TSFM_Bench",
-    "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_demo_eval.py",
-    "org": "Demo",
+    "model_link": "https://tsfm.ai/",
+    "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
+    "org": "TSFM.ai",
     "testdata_leakage": "No",
-    "replication_code_available": "Yes"
+    "replication_code_available": "Yes",
+    "api_model_id": "amazon/chronos-bolt-base"
 }

results/tsfm2/online_meta.json

@@ -0,0 +1,25 @@
+{
+    "model": "TSFM2",
+    "api_model_id": "amazon/chronos-bolt-base",
+    "evaluated_at": "2026-05-28T09:35:12.902197+00:00",
+    "datasets": [
+        {
+            "dataset": "berlin/H",
+            "data_fetched_at": "2026-05-28T09:34:12.472680+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        },
+        {
+            "dataset": "nyc/H",
+            "data_fetched_at": "2026-05-28T09:34:24.600161+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        },
+        {
+            "dataset": "tokyo/H",
+            "data_fetched_at": "2026-05-28T09:35:08.709864+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        }
+    ]
+}

results/tsfm3/all_results.csv

@@ -1,13 +1,4 @@
 dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
-synthetic_cloudops/H/short,TSFM3,0.9484441768795722,0.9484441768795722,0.9939611037572226,1.9544640077490765,0.01310331420007459,0.013185751356258216,11.959342528826262,1.1315474531752623,0.014928077942872986,0.013112953228284693,0.00977959781722658,Web/CloudOps,2
-synthetic_cloudops/H/medium,TSFM3,44.819075470547105,44.819075470547105,6.807706328272821,13.385611844716413,0.08867233581220307,0.09257877306865488,342.502721561883,7.778544329451275,0.1032732559750872,0.09038374899799995,0.08250994917856388,Web/CloudOps,2
-synthetic_energy/H/short,TSFM3,0.9432282910514187,0.9432282910514187,0.9898946590450678,1.947071336223139,0.01044440836779966,0.010496861250305302,14.977514705664005,1.1284317404785347,0.011912227550697076,0.010449768476703753,0.008438104816532673,Energy,4
-synthetic_energy/H/medium,TSFM3,44.75317610021165,44.75317610021165,6.801061638012529,13.375673774996576,0.07422846437340512,0.07696293707705479,307.0826565027702,7.772823665521155,0.08597334335215867,0.07522491600545551,0.0674160896401758,Energy,4
-synthetic_energy/H/long,TSFM3,98.1563757844154,98.1563757844154,10.04293898677826,19.75138716724768,0.1057086084767727,0.11132441580818069,551.841802671887,11.51134689378097,0.12403242298742723,0.10821062625764236,0.10037404739845796,Energy,4
-synthetic_finance/D/short,TSFM3,0.3408721604843751,0.3408721604843751,0.5517136635099139,1.9523412290537816,0.023950215287083288,0.024199645748082164,13.775353626053251,0.6783637789961271,0.02963009642749476,0.02409817498563477,0.018834635983526197,Econ/Fin,2
-synthetic_healthcare/D/short,TSFM3,1.6291463192048772,1.6291463192048772,1.2133313255310059,3.251329713548875,0.03186831168020477,0.032157332238008596,33.64701694004747,1.4830197338291167,0.03899717571702148,0.03190550592508255,0.02578070760877396,Healthcare,1
-synthetic_healthcare/D/medium,TSFM3,20.904267278593395,20.904267278593395,4.6352615927696235,12.42953198562541,0.12175143359141612,0.12921982918004207,366.85470560802383,5.312321604167154,0.14304143815887896,0.12481068238644992,0.11703214755034734,Healthcare,1
-synthetic_retail/W/short,TSFM3,0.2435628929274714,0.2435628929274714,0.46023838834762576,1.628056010173888,0.06015000478416919,0.06064468472292692,32.665706742962556,0.5734194847161077,0.07459298761744955,0.059869881157048555,0.05246497517184907,Sales,3
-synthetic_transport/15T/short,TSFM3,5.1470709532532215,5.1470709532532215,2.591809342806156,1.3498925077539736,0.022817831129293708,0.023019212182647644,4.714454848209407,2.636009443627213,0.023190008666948203,0.02280116304896367,0.015037630968304953,Transport,5
-synthetic_transport/15T/medium,TSFM3,54.907992334692004,54.907992334692004,7.775680832898617,4.049800284767568,0.06983880451833353,0.07210458851210569,85.05263573058745,8.609633537603921,0.07804941894152953,0.07048933827808697,0.06211227640099276,Transport,5
-synthetic_transport/15T/long,TSFM3,114.32069811287074,114.32069811287074,11.060530899810791,5.76069688019541,0.09643010440578481,0.10090190811537973,151.21758896689101,12.423081037020387,0.11022139545445411,0.09813243160946579,0.08988387345190475,Transport,5
+berlin/H/short,TSFM3,23.01338130609796,23.01338130609796,3.3352480199601917,0.6439920893722575,0.16218202867074588,0.1527974363779544,2.89612761434172,4.797226418056371,0.19205978695278986,0.1335286201495986,0.11504645125502568,Energy,1
+nyc/H/short,TSFM3,64.97596245257198,64.97596245257198,5.450560569763184,0.5982139619615546,0.13686922748607794,0.1192826697410579,3.5266192982165183,8.060766865042803,0.022484967353931067,0.015203972342040727,0.01133471450209458,Econ/Fin,1
+tokyo/H/short,TSFM3,9.481009614778609,9.481009614778609,1.9769952623690996,0.727519343475743,0.5269572770057526,0.8802218346383637,5.858950496884518,3.079124813121191,0.3399218079461366,0.21825156324346792,0.14708404992692353,Transport,1

results/tsfm3/config.json

@@ -2,9 +2,10 @@
     "model": "TSFM3",
     "model_type": "zero-shot",
     "model_dtype": "float32",
-    "model_link": "https://github.com/zhouziyu02/TSFM_Bench",
-    "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/generate_demo_leaderboard.py",
-    "org": "Demo",
+    "model_link": "https://tsfm.ai/",
+    "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
+    "org": "TSFM.ai",
     "testdata_leakage": "No",
-    "replication_code_available": "Yes"
+    "replication_code_available": "Yes",
+    "api_model_id": "amazon/chronos-2"
 }

results/tsfm3/online_meta.json

@@ -0,0 +1,25 @@
+{
+    "model": "TSFM3",
+    "api_model_id": "amazon/chronos-2",
+    "evaluated_at": "2026-05-28T09:35:40.814222+00:00",
+    "datasets": [
+        {
+            "dataset": "berlin/H",
+            "data_fetched_at": "2026-05-28T09:35:16.938802+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        },
+        {
+            "dataset": "nyc/H",
+            "data_fetched_at": "2026-05-28T09:35:27.945067+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        },
+        {
+            "dataset": "tokyo/H",
+            "data_fetched_at": "2026-05-28T09:35:36.696839+00:00",
+            "context_length": 504,
+            "prediction_length": 24
+        }
+    ]
+}

src/about.py

@@ -1,13 +1,14 @@
 TITLE = """ TSFM Realworld Bench Leaderboard """
 
 INTRODUCTION_TEXT = """
-**TSFM Realworld Bench** evaluates time series foundation models on diverse real-world
-multivariate forecasting tasks. Metrics and evaluation protocol follow
-[GIFT-Eval](https://huggingface.co/spaces/Salesforce/GIFT-Eval): scores are normalized
-by Seasonal Naive and aggregated with geometric mean. Primary ranking uses **CRPS_Rank**.
-
-Submit results by opening a PR to the benchmark repository with `all_results.csv` and
-`config.json` under `results/<MODEL_NAME>/`.
+**TSFM Realworld Bench** evaluates time series foundation models on **live Open-Meteo
+weather data** ([open-meteo.com](https://open-meteo.com/)) with **zero-shot API inference**
+via [TSFM.ai](https://tsfm.ai/). Metrics follow the
+[GIFT-Eval](https://huggingface.co/spaces/Salesforce/GIFT-Eval) protocol; scores are
+normalized against **TSFM2** and ranked by **CRPS_Rank**.
+
+The leaderboard auto-refreshes as new evaluations complete. Data is fetched on each run —
+no static dataset snapshot.
 """
 
 LLM_BENCHMARKS_TEXT = """
@@ -26,8 +27,15 @@ LLM_BENCHMARKS_TEXT = """
 
 ## Data interface
 
-Real datasets plug in via `TimeSeriesDataSource.stream()`. The current demo uses
-synthetic multivariate series to validate the end-to-end pipeline.
+Live weather series are streamed from **Open-Meteo** (`OpenMeteoDataSource`). Each evaluation
+holds out the most recent forecast horizon as ground truth and calls TSFM models through
+the **TSFM.ai** API for zero-shot forecasts.
+
+```bash
+export TSFM_API_KEY=your_key
+python scripts/run_online_eval.py
+python scripts/run_online_daemon.py --interval-minutes 60
+```
 """
 
 CITATION_BUTTON_LABEL = "Copy citation"