ziyuzhou02 commited on
Commit
4d9be80
·
verified ·
1 Parent(s): bfdd252

Update leaderboard results from local eval

Browse files
results/chronos_2/all_results.csv CHANGED
@@ -1,11 +1,4 @@
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
  binance_btcusdt_1h_close/1h/short,Chronos-2,420305.1332098643,420305.1332098643,488.8859049479167,0.8797728814641544,0.007336916012841026,0.007383998750081728,4.615295304962685,648.309442480876,0.009794077052542045,0.007385649366201981,0.007871857967518088,Finance,1
3
- nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,24.8303605319896,24.8303605319896,3.86064076423645,2.4106563786326287,0.16907229469696064,0.14852529859539534,60.20267861109787,4.98300717759764,0.21880485694379262,0.1695215198420369,0.1396064612336441,Weather,1
4
- noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.003406569311993124,0.003406569311993124,0.03654548029104868,0.028146189660556566,0.026838200512266448,0.02783179587963269,0.15037213100465655,0.058365823150137476,0.0396506917866611,0.024827090530161675,0.023792010233185633,Ocean,1
5
- noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-2,0.025570707400288367,0.025570707400288367,0.1241367111603419,0.8762589901319972,0.1004153818481361,0.10879610288668355,13.662508583067401,0.15990843442510583,0.13755564439698403,0.10678426912484486,0.09932759930471027,Ocean,1
6
- nws_ksfo_observations_temperature/1h/short,Chronos-2,0.921767797826457,0.921767797826457,0.7082039515177408,0.40055524970054773,0.036217152578473,0.037444525727216335,2.1820800809518897,0.9600873907236034,0.049939526836092686,0.03683765726325314,0.039891219486509366,Weather,1
7
- open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-2,765.7943036600632,765.7943036600632,20.207190195719402,3.5537804489065734,0.4647150008474858,0.6370298163721931,98.34489009739609,27.672988701259992,0.8198392159558643,0.5986576710442308,0.542720838554795,Air Quality,1
8
- open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-2,0.9301215634748891,0.9301215634748891,0.791178305943807,0.3313835954075571,0.034334453179667816,0.033970559918971376,3.044044609368773,0.9644281017654396,0.040772016812868184,0.03344773460340642,0.027221763768684827,Weather,1
9
- usgs_earthquake_all_week_hourly_earthquake_count/1h/short,Chronos-2,22.661471682037092,22.661471682037092,3.7692832946777344,0.6492545387961648,0.6405483208175075,0.3531112409323254,7.624918339925519,4.760406671917546,0.42951037641361317,0.34008571079799105,0.27618249544224943,Hazards,1
10
- usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,614.5008549491564,614.5008549491564,22.977193196614582,0.49457635580156944,0.007563419560305348,0.007578447197696511,2.586803268637892,24.78912775692514,0.008184606770755309,0.007586361765287522,0.006875891387011624,Hydrology,1
11
- wikimedia_time_series_hourly_pageviews/1D/short,Chronos-2,49.58592159347609,49.58592159347609,7.0385894775390625,0.14509077062999987,0.030940599338982695,0.030770125967900172,1.4034175379049971,7.041727174030252,0.029711929004347055,0.029698689778645832,0.04105634085739715,Web,1
 
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
  binance_btcusdt_1h_close/1h/short,Chronos-2,420305.1332098643,420305.1332098643,488.8859049479167,0.8797728814641544,0.007336916012841026,0.007383998750081728,4.615295304962685,648.309442480876,0.009794077052542045,0.007385649366201981,0.007871857967518088,Finance,1
3
+ nws_ksfo_observations_temperature/1h/short,Chronos-2,0.7954084745805782,0.7954084745805782,0.8431281248728434,0.8785126820745109,0.05582932333151499,0.05413753141501162,3.2301018126713097,0.8918567567611843,0.05848241027942192,0.05528709015559629,0.0395466244575394,Weather,1
4
+ usgs_earthquake_all_week_hourly_earthquake_count/1h/short,Chronos-2,15.124750071384748,15.124750071384748,2.954263607660929,0.612988414269703,0.760747688827298,0.31036862511166086,6.721733563228025,3.889055164353515,0.37334929577793746,0.28360930633544923,0.2397232242160373,Hazards,1
 
 
 
 
 
 
 
results/chronos_2/online_meta.json CHANGED
@@ -1,14 +1,25 @@
1
  {
2
  "model": "Chronos-2",
3
  "api_model_id": "amazon/chronos-2",
4
- "evaluated_at": "2026-06-17T01:56:32.337308+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
8
- "data_fetched_at": "2026-06-17T01:56:16.303779Z",
 
 
 
 
 
 
 
 
 
 
 
 
9
  "context_length": 96,
10
  "prediction_length": 24
11
  }
12
- ],
13
- "failed_datasets": []
14
  }
 
1
  {
2
  "model": "Chronos-2",
3
  "api_model_id": "amazon/chronos-2",
4
+ "evaluated_at": "2026-06-16T05:10:19.032951+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
8
+ "data_fetched_at": "2026-06-16T05:09:33.157863Z",
9
+ "context_length": 96,
10
+ "prediction_length": 24
11
+ },
12
+ {
13
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260616t025600z",
14
+ "data_fetched_at": "2026-06-16T05:09:33.157863Z",
15
+ "context_length": 96,
16
+ "prediction_length": 24
17
+ },
18
+ {
19
+ "dataset": "task_usgs_earthquake_all_week_hourly_earth_global_earthquake_count_20260615t050000z",
20
+ "data_fetched_at": "2026-06-16T05:09:33.157863Z",
21
  "context_length": 96,
22
  "prediction_length": 24
23
  }
24
+ ]
 
25
  }
results/dataset_properties.csv CHANGED
@@ -1,11 +1,10 @@
1
  dataset,domain,frequency,num_variates
2
  binance_btcusdt_1h_close,Finance,1h,1
 
3
  nasa_power_shanghai_hourly_t2m,Weather,1h,1
4
  noaa_coops_sf_water_level_water_level,Ocean,6min,1
5
  noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
6
  nws_ksfo_observations_temperature,Weather,1h,1
7
  open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
8
- open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
9
- usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
10
  usgs_potomac_iv_usgs_00060,Hydrology,15min,1
11
  wikimedia_time_series_hourly_pageviews,Web,1D,1
 
1
  dataset,domain,frequency,num_variates
2
  binance_btcusdt_1h_close,Finance,1h,1
3
+ coingecko_bitcoin_7d_price_usd,Finance,1h,1
4
  nasa_power_shanghai_hourly_t2m,Weather,1h,1
5
  noaa_coops_sf_water_level_water_level,Ocean,6min,1
6
  noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
7
  nws_ksfo_observations_temperature,Weather,1h,1
8
  open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 
 
9
  usgs_potomac_iv_usgs_00060,Hydrology,15min,1
10
  wikimedia_time_series_hourly_pageviews,Web,1D,1
results/online_status.json CHANGED
@@ -1,100 +1,19 @@
1
  {
2
- "status": "ok",
3
- "started_at": "2026-06-17T01:56:18.029359+00:00",
4
- "finished_at": "2026-06-17T01:59:14.692740+00:00",
5
- "task_count": 1,
6
  "data_source": "ts_bench",
7
  "data_config": "configs/datasets/ts_bench.yaml",
8
- "ts_bench_root": "/Users/bytedance/Desktop/TSFM_Bench/space/vendor/ts_bench",
9
- "models": {
10
- "Chronos-2": {
11
- "model": "Chronos-2",
12
- "api_model_id": "amazon/chronos-2",
13
- "evaluated_at": "2026-06-17T01:56:32.337308+00:00",
14
- "datasets": [
15
- {
16
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
17
- "data_fetched_at": "2026-06-17T01:56:16.303779Z",
18
- "context_length": 96,
19
- "prediction_length": 24
20
- }
21
- ],
22
- "failed_datasets": []
23
- },
24
- "TimesFM-2.5": {
25
- "model": "TimesFM-2.5",
26
- "api_model_id": "google/timesfm-2.5-200m-pytorch",
27
- "evaluated_at": "2026-06-17T01:58:07.241407+00:00",
28
- "datasets": [
29
- {
30
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
31
- "data_fetched_at": "2026-06-17T01:56:16.303779Z",
32
- "context_length": 96,
33
- "prediction_length": 24
34
- }
35
- ],
36
- "failed_datasets": []
37
- },
38
- "Toto-2.0-1B": {
39
- "model": "Toto-2.0-1B",
40
- "api_model_id": "Datadog/Toto-2.0-1B",
41
- "evaluated_at": "2026-06-17T01:58:14.524571+00:00",
42
- "datasets": [
43
- {
44
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
45
- "data_fetched_at": "2026-06-17T01:56:16.303779Z",
46
- "context_length": 96,
47
- "prediction_length": 24
48
- }
49
- ],
50
- "failed_datasets": []
51
- },
52
- "TiRex": {
53
- "model": "TiRex",
54
- "api_model_id": "NX-AI/TiRex-1.1-gifteval",
55
- "evaluated_at": "2026-06-17T01:58:54.868230+00:00",
56
- "datasets": [
57
- {
58
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
59
- "data_fetched_at": "2026-06-17T01:56:16.303779Z",
60
- "context_length": 96,
61
- "prediction_length": 24
62
- }
63
- ],
64
- "failed_datasets": []
65
- },
66
- "Moirai-1.1-Large": {
67
- "model": "Moirai-1.1-Large",
68
- "api_model_id": "Salesforce/moirai-1.1-R-large",
69
- "evaluated_at": "2026-06-17T01:59:04.905377+00:00",
70
- "datasets": [
71
- {
72
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
73
- "data_fetched_at": "2026-06-17T01:56:16.303779Z",
74
- "context_length": 96,
75
- "prediction_length": 24
76
- }
77
- ],
78
- "failed_datasets": []
79
- },
80
- "Timer-S1": {
81
- "model": "Timer-S1",
82
- "api_model_id": "bytedance-research/Timer-S1",
83
- "evaluated_at": "2026-06-17T01:59:14.689559+00:00",
84
- "datasets": [
85
- {
86
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
87
- "data_fetched_at": "2026-06-17T01:56:16.303779Z",
88
- "context_length": 96,
89
- "prediction_length": 24
90
- }
91
- ],
92
- "failed_datasets": []
93
- }
94
- },
95
- "failed_models": [],
96
- "failed_datasets": [],
97
- "push_status": "ok",
98
- "pushed_at": "2026-06-17T01:59:40.412672+00:00",
99
- "push_detail": "HF Hub API"
100
  }
 
1
  {
2
+ "status": "partial",
3
+ "started_at": "2026-06-17T00:48:23.632627+00:00",
4
+ "finished_at": "2026-06-17T02:07:31.827541+00:00",
5
+ "task_count": 9,
6
  "data_source": "ts_bench",
7
  "data_config": "configs/datasets/ts_bench.yaml",
8
+ "ts_bench_root": "/Users/zhouziyu/Desktop/TS-Bench",
9
+ "models": {},
10
+ "failed_models": [
11
+ "Chronos-Bolt-Tiny",
12
+ "Chronos-Bolt-Mini",
13
+ "Chronos-Bolt-Base",
14
+ "Chronos-2",
15
+ "TimesFM-2.5",
16
+ "Moirai-1.1-Small"
17
+ ],
18
+ "push_status": "pending"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  }
results/timesfm_2_5/all_results.csv CHANGED
@@ -1,11 +1,4 @@
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
  binance_btcusdt_1h_close/1h/short,TimesFM-2.5,1359247.1206029255,1753266.7505874634,1132.2649739583333,2.0375633836822966,0.017026736390548112,0.017226634135330692,38.68797464232892,1165.8675399044805,0.01761288016288448,0.017105242762478427,0.013654108028947516,Finance,1
3
- nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,19.47776982140628,17.26073411871069,3.198925336201986,1.9974688756129346,0.13921436982684388,0.1251515027128617,81.22673232980667,4.413362643314764,0.19379164977986701,0.14046546103892985,0.14502870025094725,Weather,1
4
- noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.007942316008117936,0.0025968019373726983,0.04441059132417043,0.03420365299327415,0.03140892311281104,0.030750095202687493,1.9657955819127375,0.08911967239682794,0.06054325068362392,0.03017023616934801,0.04132623455030458,Ocean,1
5
- noaa_ndbc_46013_realtime_wave_height/10min/short,TimesFM-2.5,0.020830764090728593,0.02700076301116816,0.1331366846958796,0.939788204397055,0.1083829007586274,0.11729325051217566,16.56187224048072,0.14432866690553403,0.12415369365927213,0.11452618194944023,0.0892202007060988,Ocean,1
6
- nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,0.216270255289146,0.38242682341357676,0.42306025822957355,0.23927995178558392,0.021540176553893703,0.022021819144783305,3.072047703072842,0.46504865905531434,0.024189787527027072,0.02200573543957979,0.017899981568634843,Weather,1
7
- open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,TimesFM-2.5,778.9301343530151,816.1337122732689,20.824244101842243,3.66229994549474,0.4682816859085522,0.6633685899858229,127.12904319713304,27.909319847552986,0.8268407561138299,0.6169384934035166,0.5872712552914257,Air Quality,1
8
- open_meteo_shanghai_hourly_temperature_2m/1h/short,TimesFM-2.5,1.1375657012913507,1.303027400657811,0.9980278015136719,0.41802213065962174,0.0419719424811146,0.042217440179163124,9.629085543135684,1.066567251180792,0.045090036071734244,0.04219247264626263,0.037096833850763754,Weather,1
9
- usgs_earthquake_all_week_hourly_earthquake_count/1h/short,TimesFM-2.5,21.473228361168214,18.106109847436453,3.338417132695516,0.5750383577848736,0.5464619941101371,0.32445871273346594,22.611880788392426,4.633921488455346,0.4180981794095049,0.3012105683635052,0.32866882620598736,Hazards,1
10
- usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,644.0699632341663,786.2911835834384,19.446889241536457,0.41858775049047087,0.006368141201098999,0.006409393150462875,16.193732702144057,25.37853351228487,0.008379210404386255,0.0064207640913038246,0.007057386604645297,Hydrology,1
11
- wikimedia_time_series_hourly_pageviews/1D/short,TimesFM-2.5,176.44351051340345,98.65426440536976,7.526611328125,0.1551506649613495,0.039470241915795116,0.038107091365422716,12.258364980081357,13.283204075576172,0.05604727458049018,0.03175785370516877,0.053793045237094536,Web,1
 
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
  binance_btcusdt_1h_close/1h/short,TimesFM-2.5,1359247.1206029255,1753266.7505874634,1132.2649739583333,2.0375633836822966,0.017026736390548112,0.017226634135330692,38.68797464232892,1165.8675399044805,0.01761288016288448,0.017105242762478427,0.013654108028947516,Finance,1
3
+ nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,0.7829723116960849,0.48572318519427427,0.6179748376210531,0.6439101199890694,0.041117021772596574,0.04006676231170827,39.74713972226133,0.8848572267298748,0.05802342470359834,0.04052294017187233,0.05512189558222573,Weather,1
4
+ usgs_earthquake_all_week_hourly_earthquake_count/1h/short,TimesFM-2.5,14.40775940274572,13.534856303621458,2.8051276206970215,0.5820437714414568,0.6715312888323447,0.30217126399252,20.592487895866633,3.795755445592579,0.3643925227768876,0.26929225158691406,0.27913609568277997,Hazards,1
 
 
 
 
 
 
 
results/timesfm_2_5/online_meta.json CHANGED
@@ -1,14 +1,25 @@
1
  {
2
  "model": "TimesFM-2.5",
3
  "api_model_id": "google/timesfm-2.5-200m-pytorch",
4
- "evaluated_at": "2026-06-17T01:58:07.241407+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
8
- "data_fetched_at": "2026-06-17T01:56:16.303779Z",
 
 
 
 
 
 
 
 
 
 
 
 
9
  "context_length": 96,
10
  "prediction_length": 24
11
  }
12
- ],
13
- "failed_datasets": []
14
  }
 
1
  {
2
  "model": "TimesFM-2.5",
3
  "api_model_id": "google/timesfm-2.5-200m-pytorch",
4
+ "evaluated_at": "2026-06-16T05:13:15.465340+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260614t230000z",
8
+ "data_fetched_at": "2026-06-16T05:09:33.157863Z",
9
+ "context_length": 96,
10
+ "prediction_length": 24
11
+ },
12
+ {
13
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260616t025600z",
14
+ "data_fetched_at": "2026-06-16T05:09:33.157863Z",
15
+ "context_length": 96,
16
+ "prediction_length": 24
17
+ },
18
+ {
19
+ "dataset": "task_usgs_earthquake_all_week_hourly_earth_global_earthquake_count_20260615t050000z",
20
+ "data_fetched_at": "2026-06-16T05:09:33.157863Z",
21
  "context_length": 96,
22
  "prediction_length": 24
23
  }
24
+ ]
 
25
  }