Spaces:
Runtime error
Runtime error
zhouziyu02 Cursor commited on
Commit ·
3da9290
1
Parent(s): c3315c0
Update demo leaderboard with TSFM1, TSFM2, and TSFM3.
Browse filesUse gr.Dataframe UI and sync synthetic demo results.
Co-authored-by: Cursor <cursoragent@cursor.com>
- README.md +7 -1
- app.py +2 -2
- requirements.txt +2 -1
- results/demo_foundation/all_results.csv +0 -13
- results/seasonal_naive/all_results.csv +0 -13
- results/tsfm1/all_results.csv +13 -0
- results/{demo_foundation → tsfm1}/config.json +5 -5
- results/tsfm2/all_results.csv +13 -0
- results/{seasonal_naive → tsfm2}/config.json +3 -3
- results/tsfm3/all_results.csv +13 -0
- results/tsfm3/config.json +10 -0
- src/utils.py +1 -1
README.md
CHANGED
|
@@ -9,4 +9,10 @@ app_file: app.py
|
|
| 9 |
pinned: false
|
| 10 |
license: apache-2.0
|
| 11 |
python_version: 3.11.9
|
| 12 |
-
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
pinned: false
|
| 10 |
license: apache-2.0
|
| 11 |
python_version: 3.11.9
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# TSFM Realworld Bench Leaderboard
|
| 15 |
+
|
| 16 |
+
Leaderboard for the **TSFM Realworld Benchmark** — evaluating time series foundation models on real-world multivariate forecasting tasks.
|
| 17 |
+
|
| 18 |
+
Results are submitted via pull request to the [TSFM_Bench](https://github.com/zhouziyu02/TSFM_Bench) repository.
|
app.py
CHANGED
|
@@ -68,7 +68,7 @@ with demo:
|
|
| 68 |
value=overall_table,
|
| 69 |
interactive=False,
|
| 70 |
wrap=True,
|
| 71 |
-
label="Overall Leaderboard (normalized vs
|
| 72 |
)
|
| 73 |
|
| 74 |
with gr.TabItem("By Domain"):
|
|
@@ -99,4 +99,4 @@ with demo:
|
|
| 99 |
show_copy_button=True,
|
| 100 |
)
|
| 101 |
|
| 102 |
-
demo.queue(default_concurrency_limit=20).launch()
|
|
|
|
| 68 |
value=overall_table,
|
| 69 |
interactive=False,
|
| 70 |
wrap=True,
|
| 71 |
+
label="Overall Leaderboard (normalized vs TSFM2, lower is better)",
|
| 72 |
)
|
| 73 |
|
| 74 |
with gr.TabItem("By Domain"):
|
|
|
|
| 99 |
show_copy_button=True,
|
| 100 |
)
|
| 101 |
|
| 102 |
+
demo.queue(default_concurrency_limit=20).launch()
|
requirements.txt
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
huggingface-hub>=0.18.0,<0.35.3
|
| 2 |
starlette==0.37.2
|
| 3 |
fastapi==0.112.4
|
|
@@ -5,4 +6,4 @@ jinja2==3.1.4
|
|
| 5 |
uvicorn==0.30.6
|
| 6 |
pandas
|
| 7 |
numpy
|
| 8 |
-
scipy
|
|
|
|
| 1 |
+
# gradio is installed by HF Space from README sdk_version — do NOT pin here
|
| 2 |
huggingface-hub>=0.18.0,<0.35.3
|
| 3 |
starlette==0.37.2
|
| 4 |
fastapi==0.112.4
|
|
|
|
| 6 |
uvicorn==0.30.6
|
| 7 |
pandas
|
| 8 |
numpy
|
| 9 |
+
scipy
|
results/demo_foundation/all_results.csv
DELETED
|
@@ -1,13 +0,0 @@
|
|
| 1 |
-
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
synthetic_cloudops/H/short,Demo_Foundation,0.9484441768795722,0.9484441768795722,0.9939611037572226,1.9544640077490765,0.013103314200074564,0.013185751356258166,11.959342528826262,1.1315474531752623,0.01492807794287292,0.013112953228284572,0.009779597817226517,Web/CloudOps,2
|
| 3 |
-
synthetic_cloudops/H/medium,Demo_Foundation,44.819075470547105,44.819075470547105,6.807706328272821,13.385611844716413,0.08867233581220305,0.09257877306865481,342.502721561883,7.778544329451275,0.10327325597508717,0.09038374899799986,0.08250994917856377,Web/CloudOps,2
|
| 4 |
-
synthetic_energy/H/short,Demo_Foundation,0.9432282910514187,0.9432282910514187,0.9898946590450678,1.947071336223139,0.010444408367799645,0.010496861250305266,14.977514705664005,1.1284317404785347,0.011912227550697062,0.010449768476703645,0.00843810481653264,Energy,4
|
| 5 |
-
synthetic_energy/H/medium,Demo_Foundation,44.75317610021165,44.75317610021165,6.801061638012529,13.375673774996578,0.07422846437340501,0.07696293707705468,307.08265650277013,7.772823665521155,0.0859733433521586,0.07522491600545549,0.06741608964017566,Energy,4
|
| 6 |
-
synthetic_energy/H/long,Demo_Foundation,98.1563757844154,98.1563757844154,10.042938986778259,19.75138716724768,0.10570860847677263,0.11132441580818057,551.841802671887,11.51134689378097,0.12403242298742721,0.1082106262576423,0.10037404739845787,Energy,4
|
| 7 |
-
synthetic_finance/D/short,Demo_Foundation,0.340872160484375,0.340872160484375,0.5517136635099139,1.9523412290537816,0.023950215287083174,0.024199645748082033,13.77535362605325,0.6783637789961271,0.0296300964274947,0.02409817498563465,0.01883463598352619,Econ/Fin,2
|
| 8 |
-
synthetic_healthcare/D/short,Demo_Foundation,1.6291463192048774,1.6291463192048774,1.2133313255310059,3.251329713548875,0.03186831168020467,0.03215733223800848,33.64701694004747,1.4830197338291167,0.03899717571702146,0.0319055059250825,0.02578070760877386,Healthcare,1
|
| 9 |
-
synthetic_healthcare/D/medium,Demo_Foundation,20.904267278593395,20.904267278593395,4.6352615927696235,12.429531985625411,0.12175143359141603,0.12921982918004202,366.85470560802383,5.312321604167154,0.1430414381588789,0.12481068238644986,0.11703214755034728,Healthcare,1
|
| 10 |
-
synthetic_retail/W/short,Demo_Foundation,0.2435628929274714,0.2435628929274714,0.46023838834762565,1.628056010173888,0.060150004784169084,0.060644684722926834,32.665706742962556,0.5734194847161077,0.07459298761744944,0.05986988115704848,0.052464975171849004,Sales,3
|
| 11 |
-
synthetic_transport/15T/short,Demo_Foundation,5.1470709532532215,5.1470709532532215,2.591809342806156,1.3498925077539734,0.02281783112929364,0.023019212182647512,4.714454848209407,2.636009443627213,0.02319000866694819,0.022801163048963596,0.015037630968304936,Transport,5
|
| 12 |
-
synthetic_transport/15T/medium,Demo_Foundation,54.907992334692004,54.907992334692004,7.775680832898617,4.049800284767568,0.06983880451833344,0.0721045885121056,85.05263573058745,8.609633537603921,0.07804941894152942,0.07048933827808684,0.06211227640099275,Transport,5
|
| 13 |
-
synthetic_transport/15T/long,Demo_Foundation,114.32069811287074,114.32069811287074,11.060530899810791,5.76069688019541,0.09643010440578481,0.10090190811537968,151.21758896689101,12.423081037020387,0.11022139545445406,0.09813243160946569,0.08988387345190472,Transport,5
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
results/seasonal_naive/all_results.csv
DELETED
|
@@ -1,13 +0,0 @@
|
|
| 1 |
-
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
synthetic_cloudops/H/short,Seasonal_Naive,0.7025512421330163,0.7025512421330163,0.7362674842646093,1.4477511168511676,0.009706158666721918,0.009767223226857937,8.85877224357501,0.8381832986483424,0.011057835513239248,0.00971329868761829,0.007244146531278948,Web/CloudOps,2
|
| 3 |
-
synthetic_cloudops/H/medium,Seasonal_Naive,33.199315163368226,33.199315163368226,5.042745428350237,9.915268033123269,0.06568321171274301,0.06857686893974435,253.70571967546886,5.761884688482426,0.07649870812969421,0.06695092518370366,0.06111848087301028,Web/CloudOps,2
|
| 4 |
-
synthetic_energy/H/short,Seasonal_Naive,0.6986876230010508,0.6986876230010508,0.7332553029963464,1.4422750638689918,0.00773659879096271,0.007775452778003927,11.094455337528892,0.8358753633174331,0.00882387225977561,0.00774056924200278,0.006250448012246424,Energy,4
|
| 5 |
-
synthetic_energy/H/medium,Seasonal_Naive,33.150500814971586,33.150500814971586,5.037823435564836,9.907906499997463,0.05498404768400379,0.05700958302004058,227.46863444649642,5.7576471596453,0.06368395803863605,0.05572216000404112,0.049937844177907996,Energy,4
|
| 6 |
-
synthetic_energy/H/long,Seasonal_Naive,72.70842650697436,72.70842650697436,7.4392140642801925,14.630657160924207,0.07830267294575755,0.08246253022828198,408.7717056828792,8.52692362502294,0.09187586887957572,0.08015601945010545,0.07435114622107997,Energy,4
|
| 7 |
-
synthetic_finance/D/short,Seasonal_Naive,0.25249789665509265,0.25249789665509265,0.4086767877851214,1.4461786881879863,0.017740900212654287,0.017925663517097897,10.203965648928333,0.5024916881452793,0.021948219575922044,0.01785049998935909,0.013951582210019405,Econ/Fin,2
|
| 8 |
-
synthetic_healthcare/D/short,Seasonal_Naive,1.2067750512628719,1.2067750512628719,0.8987639448377821,2.408392380406574,0.02360615680015168,0.02382024610222859,24.923716251887015,1.098533136169716,0.028886796827423317,0.023633708092653742,0.019096820450943675,Healthcare,1
|
| 9 |
-
synthetic_healthcare/D/medium,Seasonal_Naive,15.484642428587698,15.484642428587698,3.4335271057552763,9.207060730092895,0.09018624710475268,0.09571839198521635,271.7442263763139,3.9350530401238175,0.10595662085842886,0.09245235732329624,0.08669047966692395,Healthcare,1
|
| 10 |
-
synthetic_retail/W/short,Seasonal_Naive,0.1804169577240529,0.1804169577240529,0.34091732470194497,1.2059674149436206,0.04455555909938458,0.044921988683649564,24.19681980960189,0.4247551738637834,0.055254064901814476,0.04434806011633226,0.03886294457174005,Sales,3
|
| 11 |
-
synthetic_transport/15T/short,Seasonal_Naive,3.8126451505579415,3.8126451505579415,1.9198587724490044,0.9999203761140545,0.016902097132810153,0.017051268283442698,3.492188776451412,1.9525995878720095,0.01717778419773941,0.016889750406639755,0.011138985902448113,Transport,5
|
| 12 |
-
synthetic_transport/15T/medium,Seasonal_Naive,40.67258691458667,40.67258691458667,5.759763579924901,2.999852062790791,0.05173244779135817,0.05341080630526347,63.00195239302774,6.377506324151052,0.05781438440113298,0.05221432465043479,0.046009093630365004,Transport,5
|
| 13 |
-
synthetic_transport/15T/long,Seasonal_Naive,84.68199860212647,84.68199860212647,8.192985851711697,4.267182874218822,0.071429706967248,0.07474215415954054,112.0130288643637,9.20228224964473,0.08164547811441045,0.07269069008108577,0.06658064700141092,Transport,5
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
results/tsfm1/all_results.csv
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
synthetic_cloudops/H/short,TSFM1,0.5971685558130638,0.5971685558130638,0.6258273616249179,1.2305884493234924,0.00825023486671363,0.008302139742829246,7.529956407038758,0.712455803851091,0.00939916018625336,0.008256303884475546,0.006157524551587105,Web/CloudOps,2
|
| 3 |
+
synthetic_cloudops/H/medium,TSFM1,28.219417888862992,28.219417888862992,4.2863336140977015,8.427977828154779,0.055830729955831555,0.058290338598782694,215.64986172414854,4.897601985210062,0.06502390191024007,0.05690828640614811,0.05195070874205873,Web/CloudOps,2
|
| 4 |
+
synthetic_energy/H/short,TSFM1,0.5938844795508932,0.5938844795508932,0.6232670075468945,1.225933804288643,0.006576108972318304,0.006609134861303338,9.430287036899557,0.7104940588198181,0.007500291420809269,0.006579483855702363,0.00531288081040946,Energy,4
|
| 5 |
+
synthetic_energy/H/medium,TSFM1,28.177925692725847,28.177925692725847,4.28214992023011,8.421720524997843,0.046736440531403224,0.0484581455670345,193.34833927952195,4.894000085698505,0.054131364332840645,0.04736383600343495,0.0424471675512218,Energy,4
|
| 6 |
+
synthetic_energy/H/long,TSFM1,61.80216253092821,61.80216253092821,6.323331954638164,12.436058586785576,0.06655727200389391,0.07009315069403968,347.4559498304473,7.247885081269499,0.07809448854763935,0.06813261653258963,0.06319847428791797,Energy,4
|
| 7 |
+
synthetic_finance/D/short,TSFM1,0.21462321215682875,0.21462321215682875,0.34737526961735316,1.2292518849597884,0.015079765180756143,0.015236813989533212,8.673370801589083,0.42711793492348743,0.018655986639533738,0.015172924990955227,0.011858844878516494,Econ/Fin,2
|
| 8 |
+
synthetic_healthcare/D/short,TSFM1,1.0257587935734411,1.0257587935734411,0.7639493531121148,2.047133523345588,0.02006523328012893,0.0202472091868943,21.18515881410396,0.9337531657442585,0.024553777303309817,0.020088651878755682,0.016232297383302122,Healthcare,1
|
| 9 |
+
synthetic_healthcare/D/medium,TSFM1,13.161946064299542,13.161946064299542,2.918498039891985,7.82600162057896,0.07665831003903978,0.0813606331874339,230.98259241986682,3.344795084105245,0.09006312772966453,0.0785845037248018,0.07368690771688535,Healthcare,1
|
| 10 |
+
synthetic_retail/W/short,TSFM1,0.15335441406544495,0.15335441406544495,0.2897797259966532,1.0250723027020774,0.03787222523447689,0.03818369038110213,20.567296838161607,0.3610418977842159,0.046965955166542304,0.03769585109888242,0.03303350288597904,Sales,3
|
| 11 |
+
synthetic_transport/15T/short,TSFM1,3.24074837797425,3.24074837797425,1.6318799565816537,0.8499323196969463,0.01436678256288863,0.014493578040926292,2.9683604599837,1.659709649691208,0.014601116568078498,0.014356287845643792,0.009468138017080897,Transport,5
|
| 12 |
+
synthetic_transport/15T/medium,TSFM1,34.57169887739867,34.57169887739867,4.895799042936166,2.549874253372172,0.04397258062265444,0.04539918535947395,53.55165953407358,5.4208803755283945,0.04914222674096303,0.04438217595286957,0.039107729585810255,Transport,5
|
| 13 |
+
synthetic_transport/15T/long,TSFM1,71.9796988118075,71.9796988118075,6.964037973954942,3.6271054430859984,0.0607152509221608,0.06353083103560946,95.21107453470914,7.821939912198021,0.06939865639724888,0.0617870865689229,0.05659354995119928,Transport,5
|
results/{demo_foundation → tsfm1}/config.json
RENAMED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
{
|
| 2 |
-
"model": "
|
| 3 |
-
"model_type": "
|
| 4 |
"model_dtype": "float32",
|
| 5 |
-
"model_link": "https://
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/
|
| 7 |
-
"org": "Demo
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes"
|
| 10 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"model": "TSFM1",
|
| 3 |
+
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
+
"model_link": "https://github.com/zhouziyu02/TSFM_Bench",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/generate_demo_leaderboard.py",
|
| 7 |
+
"org": "Demo",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes"
|
| 10 |
}
|
results/tsfm2/all_results.csv
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
synthetic_cloudops/H/short,TSFM2,0.7025512421330163,0.7025512421330163,0.7362674842646093,1.4477511168511676,0.009706158666721918,0.009767223226857937,8.85877224357501,0.8381832986483424,0.011057835513239248,0.00971329868761829,0.007244146531278948,Web/CloudOps,2
|
| 3 |
+
synthetic_cloudops/H/medium,TSFM2,33.199315163368226,33.199315163368226,5.042745428350237,9.915268033123269,0.06568321171274301,0.06857686893974435,253.70571967546886,5.761884688482426,0.07649870812969421,0.06695092518370366,0.06111848087301028,Web/CloudOps,2
|
| 4 |
+
synthetic_energy/H/short,TSFM2,0.6986876230010508,0.6986876230010508,0.7332553029963464,1.4422750638689918,0.00773659879096271,0.007775452778003927,11.094455337528892,0.8358753633174331,0.00882387225977561,0.00774056924200278,0.006250448012246424,Energy,4
|
| 5 |
+
synthetic_energy/H/medium,TSFM2,33.150500814971586,33.150500814971586,5.037823435564836,9.907906499997463,0.05498404768400379,0.05700958302004058,227.46863444649642,5.7576471596453,0.06368395803863605,0.05572216000404112,0.049937844177907996,Energy,4
|
| 6 |
+
synthetic_energy/H/long,TSFM2,72.70842650697436,72.70842650697436,7.4392140642801925,14.630657160924207,0.07830267294575755,0.08246253022828198,408.7717056828792,8.52692362502294,0.09187586887957572,0.08015601945010545,0.07435114622107997,Energy,4
|
| 7 |
+
synthetic_finance/D/short,TSFM2,0.25249789665509265,0.25249789665509265,0.4086767877851214,1.4461786881879863,0.017740900212654287,0.017925663517097897,10.203965648928333,0.5024916881452793,0.021948219575922044,0.01785049998935909,0.013951582210019405,Econ/Fin,2
|
| 8 |
+
synthetic_healthcare/D/short,TSFM2,1.2067750512628719,1.2067750512628719,0.8987639448377821,2.408392380406574,0.02360615680015168,0.02382024610222859,24.923716251887015,1.098533136169716,0.028886796827423317,0.023633708092653742,0.019096820450943675,Healthcare,1
|
| 9 |
+
synthetic_healthcare/D/medium,TSFM2,15.484642428587698,15.484642428587698,3.4335271057552763,9.207060730092895,0.09018624710475268,0.09571839198521635,271.7442263763139,3.9350530401238175,0.10595662085842886,0.09245235732329624,0.08669047966692395,Healthcare,1
|
| 10 |
+
synthetic_retail/W/short,TSFM2,0.1804169577240529,0.1804169577240529,0.34091732470194497,1.2059674149436206,0.04455555909938458,0.044921988683649564,24.19681980960189,0.4247551738637834,0.055254064901814476,0.04434806011633226,0.03886294457174005,Sales,3
|
| 11 |
+
synthetic_transport/15T/short,TSFM2,3.8126451505579415,3.8126451505579415,1.9198587724490044,0.9999203761140545,0.016902097132810153,0.017051268283442698,3.492188776451412,1.9525995878720095,0.01717778419773941,0.016889750406639755,0.011138985902448113,Transport,5
|
| 12 |
+
synthetic_transport/15T/medium,TSFM2,40.67258691458667,40.67258691458667,5.759763579924901,2.999852062790791,0.05173244779135817,0.05341080630526347,63.00195239302774,6.377506324151052,0.05781438440113298,0.05221432465043479,0.046009093630365004,Transport,5
|
| 13 |
+
synthetic_transport/15T/long,TSFM2,84.68199860212647,84.68199860212647,8.192985851711697,4.267182874218822,0.071429706967248,0.07474215415954054,112.0130288643637,9.20228224964473,0.08164547811441045,0.07269069008108577,0.06658064700141092,Transport,5
|
results/{seasonal_naive → tsfm2}/config.json
RENAMED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
{
|
| 2 |
-
"model": "
|
| 3 |
-
"model_type": "
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://github.com/zhouziyu02/TSFM_Bench",
|
| 6 |
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_demo_eval.py",
|
| 7 |
-
"org": "
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes"
|
| 10 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"model": "TSFM2",
|
| 3 |
+
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://github.com/zhouziyu02/TSFM_Bench",
|
| 6 |
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_demo_eval.py",
|
| 7 |
+
"org": "Demo",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes"
|
| 10 |
}
|
results/tsfm3/all_results.csv
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
synthetic_cloudops/H/short,TSFM3,0.9484441768795722,0.9484441768795722,0.9939611037572226,1.9544640077490765,0.01310331420007459,0.013185751356258216,11.959342528826262,1.1315474531752623,0.014928077942872986,0.013112953228284693,0.00977959781722658,Web/CloudOps,2
|
| 3 |
+
synthetic_cloudops/H/medium,TSFM3,44.819075470547105,44.819075470547105,6.807706328272821,13.385611844716413,0.08867233581220307,0.09257877306865488,342.502721561883,7.778544329451275,0.1032732559750872,0.09038374899799995,0.08250994917856388,Web/CloudOps,2
|
| 4 |
+
synthetic_energy/H/short,TSFM3,0.9432282910514187,0.9432282910514187,0.9898946590450678,1.947071336223139,0.01044440836779966,0.010496861250305302,14.977514705664005,1.1284317404785347,0.011912227550697076,0.010449768476703753,0.008438104816532673,Energy,4
|
| 5 |
+
synthetic_energy/H/medium,TSFM3,44.75317610021165,44.75317610021165,6.801061638012529,13.375673774996576,0.07422846437340512,0.07696293707705479,307.0826565027702,7.772823665521155,0.08597334335215867,0.07522491600545551,0.0674160896401758,Energy,4
|
| 6 |
+
synthetic_energy/H/long,TSFM3,98.1563757844154,98.1563757844154,10.04293898677826,19.75138716724768,0.1057086084767727,0.11132441580818069,551.841802671887,11.51134689378097,0.12403242298742723,0.10821062625764236,0.10037404739845796,Energy,4
|
| 7 |
+
synthetic_finance/D/short,TSFM3,0.3408721604843751,0.3408721604843751,0.5517136635099139,1.9523412290537816,0.023950215287083288,0.024199645748082164,13.775353626053251,0.6783637789961271,0.02963009642749476,0.02409817498563477,0.018834635983526197,Econ/Fin,2
|
| 8 |
+
synthetic_healthcare/D/short,TSFM3,1.6291463192048772,1.6291463192048772,1.2133313255310059,3.251329713548875,0.03186831168020477,0.032157332238008596,33.64701694004747,1.4830197338291167,0.03899717571702148,0.03190550592508255,0.02578070760877396,Healthcare,1
|
| 9 |
+
synthetic_healthcare/D/medium,TSFM3,20.904267278593395,20.904267278593395,4.6352615927696235,12.42953198562541,0.12175143359141612,0.12921982918004207,366.85470560802383,5.312321604167154,0.14304143815887896,0.12481068238644992,0.11703214755034734,Healthcare,1
|
| 10 |
+
synthetic_retail/W/short,TSFM3,0.2435628929274714,0.2435628929274714,0.46023838834762576,1.628056010173888,0.06015000478416919,0.06064468472292692,32.665706742962556,0.5734194847161077,0.07459298761744955,0.059869881157048555,0.05246497517184907,Sales,3
|
| 11 |
+
synthetic_transport/15T/short,TSFM3,5.1470709532532215,5.1470709532532215,2.591809342806156,1.3498925077539736,0.022817831129293708,0.023019212182647644,4.714454848209407,2.636009443627213,0.023190008666948203,0.02280116304896367,0.015037630968304953,Transport,5
|
| 12 |
+
synthetic_transport/15T/medium,TSFM3,54.907992334692004,54.907992334692004,7.775680832898617,4.049800284767568,0.06983880451833353,0.07210458851210569,85.05263573058745,8.609633537603921,0.07804941894152953,0.07048933827808697,0.06211227640099276,Transport,5
|
| 13 |
+
synthetic_transport/15T/long,TSFM3,114.32069811287074,114.32069811287074,11.060530899810791,5.76069688019541,0.09643010440578481,0.10090190811537973,151.21758896689101,12.423081037020387,0.11022139545445411,0.09813243160946579,0.08988387345190475,Transport,5
|
results/tsfm3/config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "TSFM3",
|
| 3 |
+
"model_type": "zero-shot",
|
| 4 |
+
"model_dtype": "float32",
|
| 5 |
+
"model_link": "https://github.com/zhouziyu02/TSFM_Bench",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/generate_demo_leaderboard.py",
|
| 7 |
+
"org": "Demo",
|
| 8 |
+
"testdata_leakage": "No",
|
| 9 |
+
"replication_code_available": "Yes"
|
| 10 |
+
}
|
src/utils.py
CHANGED
|
@@ -105,7 +105,7 @@ def standardize_df(df: pd.DataFrame) -> pd.DataFrame:
|
|
| 105 |
df[metric] = pd.to_numeric(df[metric], errors="coerce")
|
| 106 |
df[metric] = df[metric].fillna(df[metric].mean())
|
| 107 |
|
| 108 |
-
baseline_name = "
|
| 109 |
if baseline_name not in set(df["model"]):
|
| 110 |
return df
|
| 111 |
|
|
|
|
| 105 |
df[metric] = pd.to_numeric(df[metric], errors="coerce")
|
| 106 |
df[metric] = df[metric].fillna(df[metric].mean())
|
| 107 |
|
| 108 |
+
baseline_name = "TSFM2"
|
| 109 |
if baseline_name not in set(df["model"]):
|
| 110 |
return df
|
| 111 |
|