zhouziyu02 Cursor commited on
Commit
3da9290
·
1 Parent(s): c3315c0

Update demo leaderboard with TSFM1, TSFM2, and TSFM3.

Browse files

Use gr.Dataframe UI and sync synthetic demo results.

Co-authored-by: Cursor <cursoragent@cursor.com>

README.md CHANGED
@@ -9,4 +9,10 @@ app_file: app.py
9
  pinned: false
10
  license: apache-2.0
11
  python_version: 3.11.9
12
- ---
 
 
 
 
 
 
 
9
  pinned: false
10
  license: apache-2.0
11
  python_version: 3.11.9
12
+ ---
13
+
14
+ # TSFM Realworld Bench Leaderboard
15
+
16
+ Leaderboard for the **TSFM Realworld Benchmark** — evaluating time series foundation models on real-world multivariate forecasting tasks.
17
+
18
+ Results are submitted via pull request to the [TSFM_Bench](https://github.com/zhouziyu02/TSFM_Bench) repository.
app.py CHANGED
@@ -68,7 +68,7 @@ with demo:
68
  value=overall_table,
69
  interactive=False,
70
  wrap=True,
71
- label="Overall Leaderboard (normalized vs Seasonal Naive, lower is better)",
72
  )
73
 
74
  with gr.TabItem("By Domain"):
@@ -99,4 +99,4 @@ with demo:
99
  show_copy_button=True,
100
  )
101
 
102
- demo.queue(default_concurrency_limit=20).launch()
 
68
  value=overall_table,
69
  interactive=False,
70
  wrap=True,
71
+ label="Overall Leaderboard (normalized vs TSFM2, lower is better)",
72
  )
73
 
74
  with gr.TabItem("By Domain"):
 
99
  show_copy_button=True,
100
  )
101
 
102
+ demo.queue(default_concurrency_limit=20).launch()
requirements.txt CHANGED
@@ -1,3 +1,4 @@
 
1
  huggingface-hub>=0.18.0,<0.35.3
2
  starlette==0.37.2
3
  fastapi==0.112.4
@@ -5,4 +6,4 @@ jinja2==3.1.4
5
  uvicorn==0.30.6
6
  pandas
7
  numpy
8
- scipy
 
1
+ # gradio is installed by HF Space from README sdk_version — do NOT pin here
2
  huggingface-hub>=0.18.0,<0.35.3
3
  starlette==0.37.2
4
  fastapi==0.112.4
 
6
  uvicorn==0.30.6
7
  pandas
8
  numpy
9
+ scipy
results/demo_foundation/all_results.csv DELETED
@@ -1,13 +0,0 @@
1
- dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
- synthetic_cloudops/H/short,Demo_Foundation,0.9484441768795722,0.9484441768795722,0.9939611037572226,1.9544640077490765,0.013103314200074564,0.013185751356258166,11.959342528826262,1.1315474531752623,0.01492807794287292,0.013112953228284572,0.009779597817226517,Web/CloudOps,2
3
- synthetic_cloudops/H/medium,Demo_Foundation,44.819075470547105,44.819075470547105,6.807706328272821,13.385611844716413,0.08867233581220305,0.09257877306865481,342.502721561883,7.778544329451275,0.10327325597508717,0.09038374899799986,0.08250994917856377,Web/CloudOps,2
4
- synthetic_energy/H/short,Demo_Foundation,0.9432282910514187,0.9432282910514187,0.9898946590450678,1.947071336223139,0.010444408367799645,0.010496861250305266,14.977514705664005,1.1284317404785347,0.011912227550697062,0.010449768476703645,0.00843810481653264,Energy,4
5
- synthetic_energy/H/medium,Demo_Foundation,44.75317610021165,44.75317610021165,6.801061638012529,13.375673774996578,0.07422846437340501,0.07696293707705468,307.08265650277013,7.772823665521155,0.0859733433521586,0.07522491600545549,0.06741608964017566,Energy,4
6
- synthetic_energy/H/long,Demo_Foundation,98.1563757844154,98.1563757844154,10.042938986778259,19.75138716724768,0.10570860847677263,0.11132441580818057,551.841802671887,11.51134689378097,0.12403242298742721,0.1082106262576423,0.10037404739845787,Energy,4
7
- synthetic_finance/D/short,Demo_Foundation,0.340872160484375,0.340872160484375,0.5517136635099139,1.9523412290537816,0.023950215287083174,0.024199645748082033,13.77535362605325,0.6783637789961271,0.0296300964274947,0.02409817498563465,0.01883463598352619,Econ/Fin,2
8
- synthetic_healthcare/D/short,Demo_Foundation,1.6291463192048774,1.6291463192048774,1.2133313255310059,3.251329713548875,0.03186831168020467,0.03215733223800848,33.64701694004747,1.4830197338291167,0.03899717571702146,0.0319055059250825,0.02578070760877386,Healthcare,1
9
- synthetic_healthcare/D/medium,Demo_Foundation,20.904267278593395,20.904267278593395,4.6352615927696235,12.429531985625411,0.12175143359141603,0.12921982918004202,366.85470560802383,5.312321604167154,0.1430414381588789,0.12481068238644986,0.11703214755034728,Healthcare,1
10
- synthetic_retail/W/short,Demo_Foundation,0.2435628929274714,0.2435628929274714,0.46023838834762565,1.628056010173888,0.060150004784169084,0.060644684722926834,32.665706742962556,0.5734194847161077,0.07459298761744944,0.05986988115704848,0.052464975171849004,Sales,3
11
- synthetic_transport/15T/short,Demo_Foundation,5.1470709532532215,5.1470709532532215,2.591809342806156,1.3498925077539734,0.02281783112929364,0.023019212182647512,4.714454848209407,2.636009443627213,0.02319000866694819,0.022801163048963596,0.015037630968304936,Transport,5
12
- synthetic_transport/15T/medium,Demo_Foundation,54.907992334692004,54.907992334692004,7.775680832898617,4.049800284767568,0.06983880451833344,0.0721045885121056,85.05263573058745,8.609633537603921,0.07804941894152942,0.07048933827808684,0.06211227640099275,Transport,5
13
- synthetic_transport/15T/long,Demo_Foundation,114.32069811287074,114.32069811287074,11.060530899810791,5.76069688019541,0.09643010440578481,0.10090190811537968,151.21758896689101,12.423081037020387,0.11022139545445406,0.09813243160946569,0.08988387345190472,Transport,5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
results/seasonal_naive/all_results.csv DELETED
@@ -1,13 +0,0 @@
1
- dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
- synthetic_cloudops/H/short,Seasonal_Naive,0.7025512421330163,0.7025512421330163,0.7362674842646093,1.4477511168511676,0.009706158666721918,0.009767223226857937,8.85877224357501,0.8381832986483424,0.011057835513239248,0.00971329868761829,0.007244146531278948,Web/CloudOps,2
3
- synthetic_cloudops/H/medium,Seasonal_Naive,33.199315163368226,33.199315163368226,5.042745428350237,9.915268033123269,0.06568321171274301,0.06857686893974435,253.70571967546886,5.761884688482426,0.07649870812969421,0.06695092518370366,0.06111848087301028,Web/CloudOps,2
4
- synthetic_energy/H/short,Seasonal_Naive,0.6986876230010508,0.6986876230010508,0.7332553029963464,1.4422750638689918,0.00773659879096271,0.007775452778003927,11.094455337528892,0.8358753633174331,0.00882387225977561,0.00774056924200278,0.006250448012246424,Energy,4
5
- synthetic_energy/H/medium,Seasonal_Naive,33.150500814971586,33.150500814971586,5.037823435564836,9.907906499997463,0.05498404768400379,0.05700958302004058,227.46863444649642,5.7576471596453,0.06368395803863605,0.05572216000404112,0.049937844177907996,Energy,4
6
- synthetic_energy/H/long,Seasonal_Naive,72.70842650697436,72.70842650697436,7.4392140642801925,14.630657160924207,0.07830267294575755,0.08246253022828198,408.7717056828792,8.52692362502294,0.09187586887957572,0.08015601945010545,0.07435114622107997,Energy,4
7
- synthetic_finance/D/short,Seasonal_Naive,0.25249789665509265,0.25249789665509265,0.4086767877851214,1.4461786881879863,0.017740900212654287,0.017925663517097897,10.203965648928333,0.5024916881452793,0.021948219575922044,0.01785049998935909,0.013951582210019405,Econ/Fin,2
8
- synthetic_healthcare/D/short,Seasonal_Naive,1.2067750512628719,1.2067750512628719,0.8987639448377821,2.408392380406574,0.02360615680015168,0.02382024610222859,24.923716251887015,1.098533136169716,0.028886796827423317,0.023633708092653742,0.019096820450943675,Healthcare,1
9
- synthetic_healthcare/D/medium,Seasonal_Naive,15.484642428587698,15.484642428587698,3.4335271057552763,9.207060730092895,0.09018624710475268,0.09571839198521635,271.7442263763139,3.9350530401238175,0.10595662085842886,0.09245235732329624,0.08669047966692395,Healthcare,1
10
- synthetic_retail/W/short,Seasonal_Naive,0.1804169577240529,0.1804169577240529,0.34091732470194497,1.2059674149436206,0.04455555909938458,0.044921988683649564,24.19681980960189,0.4247551738637834,0.055254064901814476,0.04434806011633226,0.03886294457174005,Sales,3
11
- synthetic_transport/15T/short,Seasonal_Naive,3.8126451505579415,3.8126451505579415,1.9198587724490044,0.9999203761140545,0.016902097132810153,0.017051268283442698,3.492188776451412,1.9525995878720095,0.01717778419773941,0.016889750406639755,0.011138985902448113,Transport,5
12
- synthetic_transport/15T/medium,Seasonal_Naive,40.67258691458667,40.67258691458667,5.759763579924901,2.999852062790791,0.05173244779135817,0.05341080630526347,63.00195239302774,6.377506324151052,0.05781438440113298,0.05221432465043479,0.046009093630365004,Transport,5
13
- synthetic_transport/15T/long,Seasonal_Naive,84.68199860212647,84.68199860212647,8.192985851711697,4.267182874218822,0.071429706967248,0.07474215415954054,112.0130288643637,9.20228224964473,0.08164547811441045,0.07269069008108577,0.06658064700141092,Transport,5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
results/tsfm1/all_results.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ synthetic_cloudops/H/short,TSFM1,0.5971685558130638,0.5971685558130638,0.6258273616249179,1.2305884493234924,0.00825023486671363,0.008302139742829246,7.529956407038758,0.712455803851091,0.00939916018625336,0.008256303884475546,0.006157524551587105,Web/CloudOps,2
3
+ synthetic_cloudops/H/medium,TSFM1,28.219417888862992,28.219417888862992,4.2863336140977015,8.427977828154779,0.055830729955831555,0.058290338598782694,215.64986172414854,4.897601985210062,0.06502390191024007,0.05690828640614811,0.05195070874205873,Web/CloudOps,2
4
+ synthetic_energy/H/short,TSFM1,0.5938844795508932,0.5938844795508932,0.6232670075468945,1.225933804288643,0.006576108972318304,0.006609134861303338,9.430287036899557,0.7104940588198181,0.007500291420809269,0.006579483855702363,0.00531288081040946,Energy,4
5
+ synthetic_energy/H/medium,TSFM1,28.177925692725847,28.177925692725847,4.28214992023011,8.421720524997843,0.046736440531403224,0.0484581455670345,193.34833927952195,4.894000085698505,0.054131364332840645,0.04736383600343495,0.0424471675512218,Energy,4
6
+ synthetic_energy/H/long,TSFM1,61.80216253092821,61.80216253092821,6.323331954638164,12.436058586785576,0.06655727200389391,0.07009315069403968,347.4559498304473,7.247885081269499,0.07809448854763935,0.06813261653258963,0.06319847428791797,Energy,4
7
+ synthetic_finance/D/short,TSFM1,0.21462321215682875,0.21462321215682875,0.34737526961735316,1.2292518849597884,0.015079765180756143,0.015236813989533212,8.673370801589083,0.42711793492348743,0.018655986639533738,0.015172924990955227,0.011858844878516494,Econ/Fin,2
8
+ synthetic_healthcare/D/short,TSFM1,1.0257587935734411,1.0257587935734411,0.7639493531121148,2.047133523345588,0.02006523328012893,0.0202472091868943,21.18515881410396,0.9337531657442585,0.024553777303309817,0.020088651878755682,0.016232297383302122,Healthcare,1
9
+ synthetic_healthcare/D/medium,TSFM1,13.161946064299542,13.161946064299542,2.918498039891985,7.82600162057896,0.07665831003903978,0.0813606331874339,230.98259241986682,3.344795084105245,0.09006312772966453,0.0785845037248018,0.07368690771688535,Healthcare,1
10
+ synthetic_retail/W/short,TSFM1,0.15335441406544495,0.15335441406544495,0.2897797259966532,1.0250723027020774,0.03787222523447689,0.03818369038110213,20.567296838161607,0.3610418977842159,0.046965955166542304,0.03769585109888242,0.03303350288597904,Sales,3
11
+ synthetic_transport/15T/short,TSFM1,3.24074837797425,3.24074837797425,1.6318799565816537,0.8499323196969463,0.01436678256288863,0.014493578040926292,2.9683604599837,1.659709649691208,0.014601116568078498,0.014356287845643792,0.009468138017080897,Transport,5
12
+ synthetic_transport/15T/medium,TSFM1,34.57169887739867,34.57169887739867,4.895799042936166,2.549874253372172,0.04397258062265444,0.04539918535947395,53.55165953407358,5.4208803755283945,0.04914222674096303,0.04438217595286957,0.039107729585810255,Transport,5
13
+ synthetic_transport/15T/long,TSFM1,71.9796988118075,71.9796988118075,6.964037973954942,3.6271054430859984,0.0607152509221608,0.06353083103560946,95.21107453470914,7.821939912198021,0.06939865639724888,0.0617870865689229,0.05659354995119928,Transport,5
results/{demo_foundation → tsfm1}/config.json RENAMED
@@ -1,10 +1,10 @@
1
  {
2
- "model": "Demo_Foundation",
3
- "model_type": "pretrained",
4
  "model_dtype": "float32",
5
- "model_link": "https://huggingface.co/ziyuzhou02/TSFM-Realworld-Bench",
6
- "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_demo_eval.py",
7
- "org": "Demo Lab",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
10
  }
 
1
  {
2
+ "model": "TSFM1",
3
+ "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
+ "model_link": "https://github.com/zhouziyu02/TSFM_Bench",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/generate_demo_leaderboard.py",
7
+ "org": "Demo",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
10
  }
results/tsfm2/all_results.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ synthetic_cloudops/H/short,TSFM2,0.7025512421330163,0.7025512421330163,0.7362674842646093,1.4477511168511676,0.009706158666721918,0.009767223226857937,8.85877224357501,0.8381832986483424,0.011057835513239248,0.00971329868761829,0.007244146531278948,Web/CloudOps,2
3
+ synthetic_cloudops/H/medium,TSFM2,33.199315163368226,33.199315163368226,5.042745428350237,9.915268033123269,0.06568321171274301,0.06857686893974435,253.70571967546886,5.761884688482426,0.07649870812969421,0.06695092518370366,0.06111848087301028,Web/CloudOps,2
4
+ synthetic_energy/H/short,TSFM2,0.6986876230010508,0.6986876230010508,0.7332553029963464,1.4422750638689918,0.00773659879096271,0.007775452778003927,11.094455337528892,0.8358753633174331,0.00882387225977561,0.00774056924200278,0.006250448012246424,Energy,4
5
+ synthetic_energy/H/medium,TSFM2,33.150500814971586,33.150500814971586,5.037823435564836,9.907906499997463,0.05498404768400379,0.05700958302004058,227.46863444649642,5.7576471596453,0.06368395803863605,0.05572216000404112,0.049937844177907996,Energy,4
6
+ synthetic_energy/H/long,TSFM2,72.70842650697436,72.70842650697436,7.4392140642801925,14.630657160924207,0.07830267294575755,0.08246253022828198,408.7717056828792,8.52692362502294,0.09187586887957572,0.08015601945010545,0.07435114622107997,Energy,4
7
+ synthetic_finance/D/short,TSFM2,0.25249789665509265,0.25249789665509265,0.4086767877851214,1.4461786881879863,0.017740900212654287,0.017925663517097897,10.203965648928333,0.5024916881452793,0.021948219575922044,0.01785049998935909,0.013951582210019405,Econ/Fin,2
8
+ synthetic_healthcare/D/short,TSFM2,1.2067750512628719,1.2067750512628719,0.8987639448377821,2.408392380406574,0.02360615680015168,0.02382024610222859,24.923716251887015,1.098533136169716,0.028886796827423317,0.023633708092653742,0.019096820450943675,Healthcare,1
9
+ synthetic_healthcare/D/medium,TSFM2,15.484642428587698,15.484642428587698,3.4335271057552763,9.207060730092895,0.09018624710475268,0.09571839198521635,271.7442263763139,3.9350530401238175,0.10595662085842886,0.09245235732329624,0.08669047966692395,Healthcare,1
10
+ synthetic_retail/W/short,TSFM2,0.1804169577240529,0.1804169577240529,0.34091732470194497,1.2059674149436206,0.04455555909938458,0.044921988683649564,24.19681980960189,0.4247551738637834,0.055254064901814476,0.04434806011633226,0.03886294457174005,Sales,3
11
+ synthetic_transport/15T/short,TSFM2,3.8126451505579415,3.8126451505579415,1.9198587724490044,0.9999203761140545,0.016902097132810153,0.017051268283442698,3.492188776451412,1.9525995878720095,0.01717778419773941,0.016889750406639755,0.011138985902448113,Transport,5
12
+ synthetic_transport/15T/medium,TSFM2,40.67258691458667,40.67258691458667,5.759763579924901,2.999852062790791,0.05173244779135817,0.05341080630526347,63.00195239302774,6.377506324151052,0.05781438440113298,0.05221432465043479,0.046009093630365004,Transport,5
13
+ synthetic_transport/15T/long,TSFM2,84.68199860212647,84.68199860212647,8.192985851711697,4.267182874218822,0.071429706967248,0.07474215415954054,112.0130288643637,9.20228224964473,0.08164547811441045,0.07269069008108577,0.06658064700141092,Transport,5
results/{seasonal_naive → tsfm2}/config.json RENAMED
@@ -1,10 +1,10 @@
1
  {
2
- "model": "Seasonal_Naive",
3
- "model_type": "statistical",
4
  "model_dtype": "float32",
5
  "model_link": "https://github.com/zhouziyu02/TSFM_Bench",
6
  "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_demo_eval.py",
7
- "org": "TSFM-Realworld-Bench",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
10
  }
 
1
  {
2
+ "model": "TSFM2",
3
+ "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://github.com/zhouziyu02/TSFM_Bench",
6
  "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_demo_eval.py",
7
+ "org": "Demo",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
10
  }
results/tsfm3/all_results.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ synthetic_cloudops/H/short,TSFM3,0.9484441768795722,0.9484441768795722,0.9939611037572226,1.9544640077490765,0.01310331420007459,0.013185751356258216,11.959342528826262,1.1315474531752623,0.014928077942872986,0.013112953228284693,0.00977959781722658,Web/CloudOps,2
3
+ synthetic_cloudops/H/medium,TSFM3,44.819075470547105,44.819075470547105,6.807706328272821,13.385611844716413,0.08867233581220307,0.09257877306865488,342.502721561883,7.778544329451275,0.1032732559750872,0.09038374899799995,0.08250994917856388,Web/CloudOps,2
4
+ synthetic_energy/H/short,TSFM3,0.9432282910514187,0.9432282910514187,0.9898946590450678,1.947071336223139,0.01044440836779966,0.010496861250305302,14.977514705664005,1.1284317404785347,0.011912227550697076,0.010449768476703753,0.008438104816532673,Energy,4
5
+ synthetic_energy/H/medium,TSFM3,44.75317610021165,44.75317610021165,6.801061638012529,13.375673774996576,0.07422846437340512,0.07696293707705479,307.0826565027702,7.772823665521155,0.08597334335215867,0.07522491600545551,0.0674160896401758,Energy,4
6
+ synthetic_energy/H/long,TSFM3,98.1563757844154,98.1563757844154,10.04293898677826,19.75138716724768,0.1057086084767727,0.11132441580818069,551.841802671887,11.51134689378097,0.12403242298742723,0.10821062625764236,0.10037404739845796,Energy,4
7
+ synthetic_finance/D/short,TSFM3,0.3408721604843751,0.3408721604843751,0.5517136635099139,1.9523412290537816,0.023950215287083288,0.024199645748082164,13.775353626053251,0.6783637789961271,0.02963009642749476,0.02409817498563477,0.018834635983526197,Econ/Fin,2
8
+ synthetic_healthcare/D/short,TSFM3,1.6291463192048772,1.6291463192048772,1.2133313255310059,3.251329713548875,0.03186831168020477,0.032157332238008596,33.64701694004747,1.4830197338291167,0.03899717571702148,0.03190550592508255,0.02578070760877396,Healthcare,1
9
+ synthetic_healthcare/D/medium,TSFM3,20.904267278593395,20.904267278593395,4.6352615927696235,12.42953198562541,0.12175143359141612,0.12921982918004207,366.85470560802383,5.312321604167154,0.14304143815887896,0.12481068238644992,0.11703214755034734,Healthcare,1
10
+ synthetic_retail/W/short,TSFM3,0.2435628929274714,0.2435628929274714,0.46023838834762576,1.628056010173888,0.06015000478416919,0.06064468472292692,32.665706742962556,0.5734194847161077,0.07459298761744955,0.059869881157048555,0.05246497517184907,Sales,3
11
+ synthetic_transport/15T/short,TSFM3,5.1470709532532215,5.1470709532532215,2.591809342806156,1.3498925077539736,0.022817831129293708,0.023019212182647644,4.714454848209407,2.636009443627213,0.023190008666948203,0.02280116304896367,0.015037630968304953,Transport,5
12
+ synthetic_transport/15T/medium,TSFM3,54.907992334692004,54.907992334692004,7.775680832898617,4.049800284767568,0.06983880451833353,0.07210458851210569,85.05263573058745,8.609633537603921,0.07804941894152953,0.07048933827808697,0.06211227640099276,Transport,5
13
+ synthetic_transport/15T/long,TSFM3,114.32069811287074,114.32069811287074,11.060530899810791,5.76069688019541,0.09643010440578481,0.10090190811537973,151.21758896689101,12.423081037020387,0.11022139545445411,0.09813243160946579,0.08988387345190475,Transport,5
results/tsfm3/config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "TSFM3",
3
+ "model_type": "zero-shot",
4
+ "model_dtype": "float32",
5
+ "model_link": "https://github.com/zhouziyu02/TSFM_Bench",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/generate_demo_leaderboard.py",
7
+ "org": "Demo",
8
+ "testdata_leakage": "No",
9
+ "replication_code_available": "Yes"
10
+ }
src/utils.py CHANGED
@@ -105,7 +105,7 @@ def standardize_df(df: pd.DataFrame) -> pd.DataFrame:
105
  df[metric] = pd.to_numeric(df[metric], errors="coerce")
106
  df[metric] = df[metric].fillna(df[metric].mean())
107
 
108
- baseline_name = "Seasonal_Naive"
109
  if baseline_name not in set(df["model"]):
110
  return df
111
 
 
105
  df[metric] = pd.to_numeric(df[metric], errors="coerce")
106
  df[metric] = df[metric].fillna(df[metric].mean())
107
 
108
+ baseline_name = "TSFM2"
109
  if baseline_name not in set(df["model"]):
110
  return df
111